{
                        config
                        for config in configs
                        if (config.host and fnmatch(host, config.host))
                        and (
                            not bucket
                            or not config.bucket
                            or fnmatch(bucket.lower(), config.bucket.lower())
                        )
                    }
                else:
                    res = {
                        config
                        for config in configs
                        if config.bucket
                        and fnmatch(bucket.lower(), config.bucket.lower())
                    }
                return next(iter(res))
            except StopIteration:
                pass

        parsed = urlparse(base_url)
        parts = Path(parsed.path.strip("/")).parts
        if parsed.netloc:
            # We have a netloc (either an actual hostname or an AWS bucket name).
            # First, we'll try with the netloc as host, but if we don't find anything, we'll try without a host and
            # with the netloc as the bucket name
            match = None
            if parts:
                # try host/bucket only if path parts contain any element
                match = find_match(host=parsed.netloc, bucket=parts[0])
            if not match:
                # no path parts or no config found for host/bucket, try netloc as bucket
                match = find_match(bucket=parsed.netloc)
        else:
            # No netloc, so we'll simply search by bucket
            match = find_match(bucket=parts[0])

        if match:
            return match

        non_aws_s3_host_suffix = ":9000"
        if parsed.netloc.endswith(non_aws_s3_host_suffix):
            host = parsed.netloc
            bucket = parts[0] if parts else None
        else:
            host = None
            bucket = parsed.netloc

        return S3BucketConfig(
            key=self.get("sdk.aws.s3.key", None),
            secret=self.get("sdk.aws.s3.secret", None),
            region=self.get("sdk.aws.s3.region", None),
            use_credentials_chain=self.get("sdk.aws.s3.use_credentials_chain", None),
            multipart=True,
            bucket=bucket,
            host=host,
        )

    def set_overrides(self, *dicts):
        """ Set several override dictionaries or ConfigTree objects which should be merged onto the configuration """
        self._overrides_configs = [
            d if isinstance(d, ConfigTree) else ConfigFactory.from_dict(d) for d in dicts
        ]
        self.reload()


# clearml/backend_config/converters.py

import base64
from distutils.util import strtobool
from typing import Union, Optional, Any, TypeVar, Callable, Tuple

import six

try:
    from typing import Text
except ImportError:
    # windows conda-less hack
    Text = Any


ConverterType = TypeVar("ConverterType", bound=Callable[[Any], Any])


def base64_to_text(value):
    # type: (Any) -> Text
    return base64.b64decode(value).decode("utf-8")


def text_to_bool(value):
    # type: (Text) -> bool
    return bool(strtobool(value))


def safe_text_to_bool(value):
    # type: (Text) -> bool
    try:
        return bool(strtobool(value))
    except ValueError:
        return bool(value)


def any_to_bool(value):
    # type: (Optional[Union[int, float, Text]]) -> bool
    if isinstance(value, six.text_type):
        return text_to_bool(value)
    return bool(value)


def or_(*converters, **kwargs):
    # type: (ConverterType, Tuple[Exception, ...]) -> ConverterType
    """
    Wrapper that implements an "optional converter" pattern. Allows specifying a converter
    for which a set of exceptions is ignored (and the original value is returned)
    :param converters: A converter callable
    :param exceptions: A tuple of exception types to ignore
    """
    # noinspection PyUnresolvedReferences
    exceptions = kwargs.get("exceptions", (ValueError, TypeError))

    def wrapper(value):
        for converter in converters:
            try:
                return converter(value)
            except exceptions:
                pass
        return value

    return wrapper


# clearml/backend_config/defs.py

from os.path import expanduser, expandvars, exists
from pathlib2 import Path

from .environment import EnvEntry


ENV_VAR = 'CLEARML_ENV'
""" Name of system environment variable that can be used to specify the config environment name """


DEFAULT_CONFIG_FOLDER = 'config'
""" Default config folder to search for when loading relative to a given path """


ENV_CONFIG_PATHS = [
]


""" Environment-related config paths """


LOCAL_CONFIG_PATHS = [
    # '/etc/opt/trains',               # used by servers for docker-generated configuration
    # expanduser('~/.trains/config'),
]
""" Local config paths, not related to environment """


LOCAL_CONFIG_FILES = [
    expanduser('~/trains.conf'),    # used for workstation configuration (end-users, workers)
    expanduser('~/clearml.conf'),    # used for workstation configuration (end-users, workers)
]
""" Local config files (not paths) """


LOCAL_CONFIG_FILE_OVERRIDE_VAR = EnvEntry("CLEARML_CONFIG_FILE", "TRAINS_CONFIG_FILE")
""" Local config file override environment variable. If this is set, no other local config files will be used. """


ENV_CONFIG_PATH_OVERRIDE_VAR = EnvEntry("CLEARML_CONFIG_PATH", "TRAINS_CONFIG_PATH")
"""
Environment-related config path override environment variable. If this is set, no other env config path will be used.
"""

CONFIG_VERBOSE = EnvEntry("CLEARML_CONFIG_VERBOSE", type=bool)


class Environment(object):
    """ Supported environment names """
    default = 'default'
    demo = 'demo'
    local = 'local'


CONFIG_FILE_EXTENSION = '.conf'


def is_config_file(path):
    return Path(path).suffix == CONFIG_FILE_EXTENSION


def get_active_config_file():
    f = LOCAL_CONFIG_FILE_OVERRIDE_VAR.get()
    if f and exists(expanduser(expandvars(f))):
        return f
    for f in LOCAL_CONFIG_FILES:
        if exists(expanduser(expandvars(f))):
            return f
    return None


def get_config_file():
    f = LOCAL_CONFIG_FILE_OVERRIDE_VAR.get()
    f = f if f else LOCAL_CONFIG_FILES[-1]
    return expanduser(expandvars(f)) if f else None


# clearml/backend_config/entry.py

import abc
from typing import Optional, Any, Tuple, Callable, Dict

import six

from .converters import any_to_bool

try:
    from typing import Text
except ImportError:
    # windows conda-less hack
    Text = Any


NotSet = object()

Converter = Callable[[Any], Any]


@six.add_metaclass(abc.ABCMeta)
class Entry(object):
    """
    Configuration entry definition
    """

    @classmethod
    def default_conversions(cls):
        # type: () -> Dict[Any, Converter]
        return {
            bool: any_to_bool,
            six.text_type: lambda s: six.text_type(s).strip(),
        }

    def __init__(self, key, *more_keys, **kwargs):
        # type: (Text, Text, Any) -> None
        """
        :param key: Entry's key (at least one).
        :param more_keys: More alternate keys for this entry.
        :param type: Value type. If provided, will be used choosing a default conversion or
        (if none exists) for casting the environment value.
        :param converter: Value converter. If provided, will be used to convert the environment value.
        :param default: Default value. If provided, will be used as the default value on calls to get() and get_pair()
        in case no value is found for any key and no specific default value was provided in the call.
        Default value is None.
        :param help: Help text describing this entry
        """
        self.keys = (key,) + more_keys
        self.type = kwargs.pop("type", six.text_type)
        self.converter = kwargs.pop("converter", None)
        self.default = kwargs.pop("default", None)
        self.help = kwargs.pop("help", None)

    def __str__(self):
        return str(self.key)

    @property
    def key(self):
        return self.keys[0]

    def convert(self, value, converter=None):
        # type: (Any, Converter) -> Optional[Any]
        converter = converter or self.converter
        if not converter:
            converter = self.default_conversions().get(self.type, self.type)
        return converter(value)

    def get_pair(self, default=NotSet, converter=None):
        # type: (Any, Converter) -> Optional[Tuple[Text, Any]]
        for key in self.keys:
            value = self._get(key)
            if value is NotSet:
                continue
            # noinspection PyBroadException
            try:
                value = self.convert(value, converter)
            except Exception as ex:  # noqa: F841
                self.error("invalid value {key}={value}: {ex}".format(**locals()))
                break
            return key, value
        result = self.default if default is NotSet else default
        return self.key, result

    def get(self, default=NotSet, converter=None):
        # type: (Any, Converter) -> Optional[Any]
        return self.get_pair(default=default, converter=converter)[1]

    def set(self, value):
        # type: (Any) -> ()
        # key, _ = self.get_pair(default=None, converter=None)
        for k in self.keys:
            self._set(k, str(value))

    def _set(self, key, value):
        # type: (Text, Text) -> None
        pass

    @abc.abstractmethod
    def _get(self, key):
        # type: (Text) -> Any
        pass

    @abc.abstractmethod
    def error(self, message):
        # type: (Text) -> None
        pass

    def exists(self):
        # type: () -> bool
        return any(key for key in self.keys if self._get(key) is not NotSet)


# clearml/backend_config/environment.py

from os import getenv, environ

from .converters import text_to_bool
from .entry import Entry, NotSet


class EnvEntry(Entry):
    @classmethod
    def default_conversions(cls):
        conversions = super(EnvEntry, cls).default_conversions().copy()
        conversions[bool] = text_to_bool
        return conversions

    def __init__(self, key, *more_keys, **kwargs):
        super(EnvEntry, self).__init__(key, *more_keys, **kwargs)
        self._ignore_errors = kwargs.pop('ignore_errors', False)

    def pop(self):
        for k in self.keys:
            environ.pop(k, None)

    def _get(self, key):
        value = getenv(key, "").strip()
        return value or NotSet

    def _set(self, key, value):
        environ[key] = value

    def __str__(self):
        return "env:{}".format(super(EnvEntry, self).__str__())

    def error(self, message):
        if not self._ignore_errors:
            print("Environment configuration: {}".format(message))

    def exists(self):
        return any(key for key in self.keys if getenv(key) is not None)


# clearml/backend_config/errors.py

class ConfigurationError(Exception):

    def __init__(self, msg, file_path=None, *args):
        super(ConfigurationError, self).__init__(msg, *args)
        self.file_path = file_path


# clearml/backend_config/log.py

import logging.config
from copy import deepcopy

from pathlib2 import Path


def logger(path=None):
    name = "clearml"
    if path:
        p = Path(path)
        module = (p.parent if p.stem.startswith('_') else p).stem
        name = "clearml.%s" % module
    return logging.getLogger(name)


def initialize(logging_config=None, extra=None):
    if extra is not None:
        from logging import Logger

        class _Logger(Logger):
            __extra = extra.copy()

            def _log(self, level, msg, args, exc_info=None, extra=None, **kwargs):
                extra = extra or {}
                extra.update(self.__extra)
                super(_Logger, self)._log(level, msg, args, exc_info=exc_info, extra=extra, **kwargs)

        Logger.manager.loggerClass = _Logger

    if logging_config is not None:
        # Use deepcopy since Python's logging infrastructure might modify the dict
        logging.config.dictConfig(deepcopy(dict(logging_config)))


# clearml/backend_config/utils.py

import base64
import os
from os.path import expandvars, expanduser
from pathlib import Path
from typing import List, TYPE_CHECKING

from ..utilities.pyhocon import HOCONConverter, ConfigTree

if TYPE_CHECKING:
    from .config import Config


def get_items(cls):
    """ get key/value items from an enum-like class (members represent enumeration key/value) """
    return {k: v for k, v in vars(cls).items() if not k.startswith('_')}


def get_options(cls):
    """ get options from an enum-like class (members represent enumeration key/value) """
    return get_items(cls).values()


def apply_environment(config):
    # type: (Config) -> List[str]
    env_vars = config.get("environment", None)
    if not env_vars:
        return []
    if isinstance(env_vars, (list, tuple)):
        env_vars = dict(env_vars)

    keys = list(filter(None, env_vars.keys()))

    for key in keys:
        os.environ[str(key)] = str(env_vars[key] or "")

    return keys


def apply_files(config):
    # type: (Config) -> None
    files = config.get("files", None)
    if not files:
        return

    if isinstance(files, (list, tuple)):
        files = dict(files)

    print("Creating files from configuration")
    for key, data in files.items():
        path = data.get("path")
        fmt = data.get("format", "string")
        target_fmt = data.get("target_format", "string")
        overwrite = bool(data.get("overwrite", True))
        contents = data.get("contents")

        target = Path(expanduser(expandvars(path)))

        # noinspection PyBroadException
        try:
            if target.is_dir():
                print("Skipped [{}]: is a directory {}".format(key, target))
                continue

            if not overwrite and target.is_file():
                print("Skipped [{}]: file exists {}".format(key, target))
                continue
        except Exception as ex:
            print("Skipped [{}]: can't access {} ({})".format(key, target, ex))
            continue

        if contents:
            try:
                if fmt == "base64":
                    contents = base64.b64decode(contents)
                    if target_fmt != "bytes":
                        contents = contents.decode("utf-8")
            except Exception as ex:
                print("Skipped [{}]: failed decoding {} ({})".format(key, fmt, ex))
                continue

        # noinspection PyBroadException
        try:
            target.parent.mkdir(parents=True, exist_ok=True)
        except Exception as ex:
            print("Skipped [{}]: failed creating path {} ({})".format(key, target.parent, ex))
            continue

        try:
            if target_fmt == "bytes":
                try:
                    target.write_bytes(contents)
                except TypeError:
                    # simpler error so the user won't get confused
                    raise TypeError("a bytes-like object is required")
            else:
                try:
                    if target_fmt == "json":
                        text = HOCONConverter.to_json(contents)
                    elif target_fmt in ("yaml", "yml"):
                        text = HOCONConverter.to_yaml(contents)
                    else:
                        if isinstance(contents, ConfigTree):
                            contents = contents.as_plain_ordered_dict()
                        text = str(contents)
                except Exception as ex:
                    print("Skipped [{}]: failed encoding to {} ({})".format(key, target_fmt, ex))
                    continue
                target.write_text(text)
            print("Saved [{}]: {}".format(key, target))
        except Exception as ex:
            print("Skipped [{}]: failed saving file {} ({})".format(key, target, ex))
            continue


# clearml/backend_interface/__init__.py

""" High-level abstractions for backend API """
from .task import Task

__all__ = ["Task"]


# clearml/backend_interface/base.py

import abc

import requests.exceptions
import six
import jsonschema

from ..backend_api import Session, CallResult
from ..backend_api.session.session import MaxRequestSizeError
from ..backend_api.session.response import ResponseMeta
from ..backend_api.session import BatchRequest
from ..backend_api.session.defs import ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_OFFLINE_MODE

from ..config import config_obj
from ..config.defs import LOG_LEVEL_ENV_VAR
from ..debugging import get_logger
from .session import SendError, SessionInterface


class ValidationError(ValueError):
    pass


class InterfaceBase(SessionInterface):
    """ Base class for a backend manager class """
    _default_session = None
    _num_retry_warning_display = 1
    _offline_mode = ENV_OFFLINE_MODE.get()

    @property
    def session(self):
        return self._session

    @property
    def log(self):
        return self._log

    def __init__(self, session=None, log=None, **kwargs):
        super(InterfaceBase, self).__init__()
        self._session = session or self._get_default_session()
        self._log = log or self._create_log()

    def _create_log(self):
        log = get_logger(str(self.__class__.__name__))
        try:
            log.setLevel(LOG_LEVEL_ENV_VAR.get(default=log.level))
        except TypeError as ex:
            raise ValueError('Invalid log level defined in environment variable `%s`: %s' % (LOG_LEVEL_ENV_VAR, ex))
        return log

    @classmethod
    def _send(cls, session, req, ignore_errors=False, raise_on_errors=True, log=None, async_enable=False):
        """ Convenience send() method providing a standardized error reporting """
        if cls._offline_mode:
            return None

        num_retries = 0
        while True:
            error_msg = ''
            try:
                res = session.send(req, async_enable=async_enable)
                if res.meta.result_code in (200, 202) or ignore_errors:
                    return res

                if isinstance(req, BatchRequest):
                    error_msg = 'Action failed %s' % res.meta
                else:
                    error_msg = 'Action failed %s (%s)' \
                                % (res.meta, ', '.join('%s=%s' % p for p in req.to_dict().items()))
                if log:
                    log.error(error_msg)

            except requests.exceptions.BaseHTTPError as e:
                res = None
                if log and num_retries >= cls._num_retry_warning_display:
                    log.warning('Retrying, previous request failed %s: %s' % (str(type(req)), str(e)))
            except MaxRequestSizeError as e:
                res = CallResult(meta=ResponseMeta.from_raw_data(status_code=400, text=str(e)))
                error_msg = 'Failed sending: %s' % str(e)
            except requests.exceptions.ConnectionError:
                # We couldn't send the request for more than the retries times configure in the api configuration file,
                # so we will end the loop and raise the exception to the upper level.
                # Notice: this is a connectivity error and not a backend error.
                if raise_on_errors:
                    raise
                res = None
            except jsonschema.ValidationError as e:
                if log:
                    log.error(
                        'Field %s contains illegal schema: %s', '.'.join(e.path), str(e.message)
                    )
                if raise_on_errors:
                    raise ValidationError("Field %s contains illegal schema: %s" % ('.'.join(e.path), e.message))
                # We do not want to retry
                return None
            except Exception as e:
                res = None
                if log and num_retries >= cls._num_retry_warning_display:
                    log.warning('Retrying, previous request failed %s: %s' % (str(type(req)), str(e)))

            if res and res.meta.result_code <= 500:
                # Proper backend error/bad status code - raise or return
                if raise_on_errors:
                    raise SendError(res, error_msg)
                return res

            num_retries += 1

    def send(self, req, ignore_errors=False, raise_on_errors=True, async_enable=False):
        return self._send(session=self.session, req=req, ignore_errors=ignore_errors, raise_on_errors=raise_on_errors,
                          log=self.log, async_enable=async_enable)

    @classmethod
    def _get_default_session(cls):
        if not InterfaceBase._default_session:
            InterfaceBase._default_session = Session(
                initialize_logging=False,
                config=config_obj,
                api_key=ENV_ACCESS_KEY.get(),
                secret_key=ENV_SECRET_KEY.get(),
            )
        return InterfaceBase._default_session

    @classmethod
    def _set_default_session(cls, session):
        """
        Set a new default session to the system

        Warning: Use only for debug and testing
        :param session: The new default session
        """

        InterfaceBase._default_session = session

    @property
    def default_session(self):
        if hasattr(self, '_session'):
            return self._session
        return self._get_default_session()


@six.add_metaclass(abc.ABCMeta)
class IdObjectBase(InterfaceBase):

    def __init__(self, id, session=None, log=None, **kwargs):
        super(IdObjectBase, self).__init__(session, log, **kwargs)
        self._data = None
        self._id = None
        self.id = self.normalize_id(id)

    @property
    def id(self):
        return self._id

    @id.setter
    def id(self, value):
        should_reload = value is not None and self._id is not None and value != self._id
        self._id = value
        if should_reload:
            self.reload()

    @property
    def data(self):
        if self._data is None:
            self.reload()
        return self._data

    @abc.abstractmethod
    def _reload(self):
        pass

    def reload(self):
        if not self.id and not self._offline_mode:
            raise ValueError('Failed reloading %s: missing id' % type(self).__name__)
        # noinspection PyBroadException
        try:
            self._data = self._reload()
        except Exception:
            self.log.error("Failed reloading task {}".format(self.id))

    @classmethod
    def normalize_id(cls, id):
        return id.strip() if id else None

    @classmethod
    def resolve_id(cls, obj):
        if isinstance(obj, cls):
            return obj.id
        return obj


# clearml/backend_interface/logger.py

import logging
import sys
import threading
from time import time

from ..binding.frameworks import _patched_call  # noqa
from ..config import running_remotely, config, DEBUG_SIMULATE_REMOTE_TASK


class StdStreamPatch(object):
    _stdout_proxy = None
    _stderr_proxy = None
    _stdout_original_write = None
    _stderr_original_write = None

    @staticmethod
    def patch_std_streams(a_logger, connect_stdout=True, connect_stderr=True):
        if (connect_stdout or connect_stderr) and not PrintPatchLogger.patched and \
                (not running_remotely() or DEBUG_SIMULATE_REMOTE_TASK.get()):
            StdStreamPatch._stdout_proxy = PrintPatchLogger(sys.stdout, a_logger, level=logging.INFO) \
                if connect_stdout else None
            StdStreamPatch._stderr_proxy = PrintPatchLogger(sys.stderr, a_logger, level=logging.ERROR) \
                if connect_stderr else None

            if StdStreamPatch._stdout_proxy:
                # noinspection PyBroadException
                try:
                    if StdStreamPatch._stdout_original_write is None:
                        StdStreamPatch._stdout_original_write = sys.stdout.write

                    # this will only work in python 3, guard it with try/catch
                    if not hasattr(sys.stdout, '_original_write'):
                        sys.stdout._original_write = sys.stdout.write
                    sys.stdout.write = StdStreamPatch._stdout__patched__write__
                except Exception:
                    pass
                sys.stdout = StdStreamPatch._stdout_proxy
                # noinspection PyBroadException
                try:
                    sys.__stdout__ = sys.stdout
                except Exception:
                    pass

            if StdStreamPatch._stderr_proxy:
                # noinspection PyBroadException
                try:
                    if StdStreamPatch._stderr_original_write is None:
                        StdStreamPatch._stderr_original_write = sys.stderr.write
                    if not hasattr(sys.stderr, '_original_write'):
                        sys.stderr._original_write = sys.stderr.write
                    sys.stderr.write = StdStreamPatch._stderr__patched__write__
                except Exception:
                    pass
                sys.stderr = StdStreamPatch._stderr_proxy

                # patch the base streams of sys (this way colorama will keep its ANSI colors)
                # noinspection PyBroadException
                try:
                    sys.__stderr__ = sys.stderr
                except Exception:
                    pass

            # now check if we have loguru and make it re-register the handlers
            # because it stores internally the stream.write function, which we cant patch
            # noinspection PyBroadException
            try:
                from loguru import logger  # noqa
                register_stderr = None
                register_stdout = None
                for k, v in logger._handlers.items():  # noqa
                    if connect_stderr and v._name == '<stderr>':  # noqa
                        register_stderr = k
                    elif connect_stdout and v._name == '<stdout>':  # noqa
                        register_stderr = k
                if register_stderr is not None:
                    logger.remove(register_stderr)
                    logger.add(sys.stderr)
                if register_stdout is not None:
                    logger.remove(register_stdout)
                    logger.add(sys.stdout)
            except Exception:
                pass

        elif (connect_stdout or connect_stderr) and not running_remotely():
            if StdStreamPatch._stdout_proxy and connect_stdout:
                StdStreamPatch._stdout_proxy.connect(a_logger)
            if StdStreamPatch._stderr_proxy and connect_stderr:
                StdStreamPatch._stderr_proxy.connect(a_logger)

    @staticmethod
    def patch_logging_formatter(a_logger, logging_handler=None):
        if not logging_handler:
            import logging
            logging_handler = logging.Handler
        logging_handler.format = _patched_call(logging_handler.format, HandlerFormat(a_logger))

    @staticmethod
    def remove_patch_logging_formatter(logging_handler=None):
        if not logging_handler:
            import logging
            logging_handler = logging.Handler
        # remove the function, Hack calling patched logging.Handler.format() returns the original function
        # noinspection PyBroadException
        try:
            logging_handler.format = logging_handler.format()  # noqa
        except Exception:
            pass

    @staticmethod
    def remove_std_logger(logger=None):
        if isinstance(sys.stdout, PrintPatchLogger):
            # noinspection PyBroadException
            try:
                sys.stdout.disconnect(logger)
            except Exception:
                pass
        if isinstance(sys.stderr, PrintPatchLogger):
            # noinspection PyBroadException
            try:
                sys.stderr.disconnect(logger)
            except Exception:
                pass

    @staticmethod
    def stdout_original_write(*args, **kwargs):
        if StdStreamPatch._stdout_original_write:
            StdStreamPatch._stdout_original_write(*args, **kwargs)
        else:
            sys.stdout.write(*args, **kwargs)

    @staticmethod
    def stderr_original_write(*args, **kwargs):
        if StdStreamPatch._stderr_original_write:
            StdStreamPatch._stderr_original_write(*args, **kwargs)
        else:
            sys.stderr.write(*args, **kwargs)

    @staticmethod
    def _stdout__patched__write__(*args, **kwargs):
        if StdStreamPatch._stdout_proxy:
            return StdStreamPatch._stdout_proxy.write(*args, **kwargs)
        return sys.stdout._original_write(*args, **kwargs)  # noqa

    @staticmethod
    def _stderr__patched__write__(*args, **kwargs):
        if StdStreamPatch._stderr_proxy:
            return StdStreamPatch._stderr_proxy.write(*args, **kwargs)
        return sys.stderr._original_write(*args, **kwargs)  # noqa


class HandlerFormat(object):
    def __init__(self, logger):
        self._logger = logger

    def __call__(self, original_format_func, *args):
        # hack get back original function, so we can remove it
        if all(a is None for a in args):
            return original_format_func
        if len(args) == 1:
            record = args[0]
            msg = original_format_func(record)
        else:
            handler = args[0]
            record = args[1]
            msg = original_format_func(handler, record)

        self._logger.report_text(msg=msg, level=record.levelno, print_console=False)
        return msg


class PrintPatchLogger(object):
    """
    Allowed patching a stream into the logger.
    Used for capturing and logging stdin and stderr when running in development mode pseudo worker.
    """
    patched = False
    lock = threading.Lock()
    recursion_protect_lock = threading.RLock()
    cr_flush_period = None

    def __init__(self, stream, logger=None, level=logging.INFO):
        if PrintPatchLogger.cr_flush_period is None:
            PrintPatchLogger.cr_flush_period = config.get("development.worker.console_cr_flush_period", 0)
        PrintPatchLogger.patched = True
        self._terminal = stream
        self._log = logger
        self._log_level = level
        self._cur_line = ''
        self._force_lf_flush = False
        self._lf_last_flush = 0

    def write(self, message):
        # make sure that we do not end up in infinite loop (i.e. log.console ends up calling us)
        if self._log and not PrintPatchLogger.recursion_protect_lock._is_owned():  # noqa
            try:
                # make sure we flush from time to time on \r
                self._test_lr_flush()

                self.lock.acquire()
                with PrintPatchLogger.recursion_protect_lock:
                    if hasattr(self._terminal, '_original_write'):
                        self._terminal._original_write(message)  # noqa
                    else:
                        self._terminal.write(message)

                do_flush = '\n' in message
                # check for CR character
                do_cr = '\r' in message
                # check for "Escape Arrow-Up" character (tqdm's way of clearing a line)
                if '\x1b[A' in message:
                    do_cr = True
                    # replace it with \r so it is more standard
                    message = message.replace('\x1b[A', '\r')

                self._cur_line += message

                if not do_flush and do_cr and PrintPatchLogger.cr_flush_period and self._force_lf_flush:
                    self._cur_line += '\n'
                    do_flush = True

                if (not do_flush and (PrintPatchLogger.cr_flush_period or not do_cr)) or not message:
                    return

                if PrintPatchLogger.cr_flush_period and self._cur_line:
                    self._cur_line = '\n'.join(line.split('\r')[-1] for line in self._cur_line.split('\n'))

                last_lf = self._cur_line.rindex('\n' if do_flush else '\r')
                next_line = self._cur_line[last_lf + 1:]
                cur_line = self._cur_line[:last_lf + 1].rstrip()
                self._cur_line = next_line
            finally:
                self.lock.release()

            if cur_line:
                self._force_lf_flush = False
                with PrintPatchLogger.recursion_protect_lock:
                    # noinspection PyBroadException
                    try:
                        if self._log:
                            # noinspection PyProtectedMember
                            self._log._console(cur_line, level=self._log_level, omit_console=True)
                    except Exception:
                        # what can we do, nothing
                        pass
        else:
            if hasattr(self._terminal, '_original_write'):
                self._terminal._original_write(message)  # noqa
            else:
                self._terminal.write(message)

    def connect(self, logger):
        self._cur_line = ''
        self._log = logger

    def disconnect(self, logger=None):
        # disconnect the logger only if it was registered
        if not logger or self._log == logger:
            self.connect(None)

    def _test_lr_flush(self):
        if not self.cr_flush_period:
            return
        if time() - self._lf_last_flush > self.cr_flush_period:
            self._force_lf_flush = True
            self._lf_last_flush = time()

    def __getattr__(self, attr):
        if attr in ['_log', '_terminal', '_log_level', '_cur_line', '_cr_overwrite',
                    '_force_lf_flush', '_lf_last_flush']:
            return self.__dict__.get(attr)
        return getattr(self._terminal, attr)

    def __setattr__(self, key, value):
        if key in ['_log', '_terminal', '_log_level', '_cur_line', '_cr_overwrite',
                   '_force_lf_flush', '_lf_last_flush']:
            self.__dict__[key] = value
        else:
            return setattr(self._terminal, key, value)


# clearml/backend_interface/model.py

from collections import namedtuple
from functools import partial

import six
from pathlib2 import Path

from .base import IdObjectBase
from .util import make_message
from ..backend_api import Session
from ..backend_api.services import models, tasks
from ..storage import StorageManager
from ..storage.helper import StorageHelper
from ..utilities.async_manager import AsyncManagerMixin

ModelPackage = namedtuple('ModelPackage', 'weights design')


class ModelDoesNotExistError(Exception):
    pass


class _StorageUriMixin(object):
    @property
    def upload_storage_uri(self):
        """ A URI into which models are uploaded """
        return self._upload_storage_uri

    @upload_storage_uri.setter
    def upload_storage_uri(self, value):
        self._upload_storage_uri = value.rstrip('/') if value else None


def create_dummy_model(upload_storage_uri=None, *args, **kwargs):
    class DummyModel(models.Model, _StorageUriMixin):
        def __init__(self, upload_storage_uri=None, *_, **__):
            super(DummyModel, self).__init__(*_, **__)
            self.upload_storage_uri = upload_storage_uri

        def update(self, **a_kwargs):
            for k, v in a_kwargs.items():
                setattr(self, k, v)

    return DummyModel(upload_storage_uri=upload_storage_uri, *args, **kwargs)


class Model(IdObjectBase, AsyncManagerMixin, _StorageUriMixin):
    """ Manager for backend model objects """

    _EMPTY_MODEL_ID = 'empty'

    _local_model_to_id_uri = {}

    @property
    def model_id(self):
        return self.id

    def __init__(self, upload_storage_uri, cache_dir, model_id=None,
                 upload_storage_suffix='models', session=None, log=None):
        super(Model, self).__init__(id=model_id, session=session, log=log)
        self._upload_storage_suffix = upload_storage_suffix
        if model_id == self._EMPTY_MODEL_ID:
            # Set an empty data object
            self._data = models.Model()
        else:
            self._data = None
        self._cache_dir = cache_dir
        self.upload_storage_uri = upload_storage_uri

    def publish(self):
        self.send(models.SetReadyRequest(model=self.id, publish_task=False))
        self.reload()

    def _reload(self):
        """ Reload the model object """
        if self._offline_mode:
            return models.Model()

        if self.id == self._EMPTY_MODEL_ID:
            return
        res = self.send(models.GetByIdRequest(model=self.id))
        return res.response.model

    def _upload_model(self, model_file, async_enable=False, target_filename=None, cb=None):
        if not self.upload_storage_uri:
            raise ValueError('Model has no storage URI defined (nowhere to upload to)')
        target_filename = target_filename or Path(model_file).name
        dest_path = '/'.join((self.upload_storage_uri, self._upload_storage_suffix or '.', target_filename))
        result = StorageHelper.get(dest_path).upload(
            src_path=model_file,
            dest_path=dest_path,
            async_enable=async_enable,
            cb=partial(self._upload_callback, cb=cb),
        )
        if async_enable:
            def msg(num_results):
                self.log.info("Waiting for previous model to upload (%d pending, %s)" % (num_results, dest_path))

            self._add_async_result(result, wait_on_max_results=2, wait_cb=msg)
        return dest_path

    def _upload_callback(self, res, cb=None):
        if res is None:
            self.log.debug('Starting model upload')
        elif res is False:
            self.log.info('Failed model upload')
        else:
            self.log.info('Completed model upload to {}'.format(res))
        if cb:
            cb(res)

    @staticmethod
    def _wrap_design(design):
        """
        Wrap design text with a dictionary.

        In the backend, the design is a dictionary with a 'design' key in it.
        For the client, it is a text. This function wraps a design string with
        the proper dictionary.

        :param design: If it is a dictionary, it mast have a 'design' key in it.
            In that case, return design as-is.
            If it is a string, return the dictionary {'design': design}.
            If it is None (or any False value), return the dictionary {'design': ''}

        :return: A proper design dictionary according to design parameter.
        """
        if isinstance(design, dict):
            if 'design' not in design:
                raise ValueError('design dictionary must have \'design\' key in it')

            return design

        return {'design': design if design else ''}

    @staticmethod
    def _unwrap_design(design):
        """
        Unwrap design text from a dictionary.

        In the backend, the design is a dictionary with a 'design' key in it.
        For the client, it is a text. This function unwraps a design string from
        the dictionary.

        :param design: If it is a dictionary with a 'design' key in it, return
            design['design'].
            If it is a dictionary without 'design' key, return the first value
            in it's values list.
            If it is an empty dictionary, None, or any other False value,
            return an empty string.
            If it is a string, return design as-is.

        :return: The design string according to design parameter.
        """
        if not design:
            return ''

        if isinstance(design, six.string_types):
            return design

        if isinstance(design, dict):
            if 'design' in design:
                return design['design']

            return list(design.values())[0]

        raise ValueError('design must be a string or a dictionary with at least one value')

    def update(self, model_file=None, design=None, labels=None, name=None, comment=None, tags=None,
               task_id=None, project_id=None, parent_id=None, uri=None, framework=None,
               upload_storage_uri=None, target_filename=None, iteration=None, system_tags=None):
        """ Update model weights file and various model properties """

        if self.id is None:
            if upload_storage_uri:
                self.upload_storage_uri = upload_storage_uri
            self._create_empty_model(self.upload_storage_uri)
        elif upload_storage_uri:
            self.upload_storage_uri = upload_storage_uri

        if model_file and uri:
            Model._local_model_to_id_uri[str(model_file)] = (self.model_id, uri)

        # upload model file if needed and get uri
        uri = uri or (self._upload_model(model_file, target_filename=target_filename) if model_file else self.data.uri)
        # update fields
        design = self._wrap_design(design) if design else self.data.design
        name = name or self.data.name
        comment = comment or self.data.comment
        labels = labels or self.data.labels
        task = task_id or self.data.task
        project = project_id or self.data.project
        parent = parent_id or self.data.parent
        tags = tags or self.data.tags
        if Session.check_min_api_version('2.3'):
            system_tags = system_tags or self.data.system_tags

        self._edit(
            uri=uri,
            name=name,
            comment=comment,
            labels=labels,
            design=design,
            framework=framework or self.data.framework,
            iteration=iteration,
            task=task,
            project=project,
            parent=parent,
            tags=tags,
            system_tags=system_tags,
        )

    def edit(self, design=None, labels=None, name=None, comment=None, tags=None,
             uri=None, framework=None, iteration=None, system_tags=None):
        return self._edit(design=design, labels=labels, name=name, comment=comment, tags=tags,
                          uri=uri, framework=framework, iteration=iteration, system_tags=system_tags)

    def _edit(self, design=None, labels=None, name=None, comment=None, tags=None,
              uri=None, framework=None, iteration=None, system_tags=None, **extra):
        def offline_store(**kwargs):
            for k, v in kwargs.items():
                setattr(self.data, k, v or getattr(self.data, k, None))
            return
        if self._offline_mode:
            return offline_store(design=design, labels=labels, name=name, comment=comment, tags=tags,
                                 uri=uri, framework=framework, iteration=iteration, **extra)

        if Session.check_min_api_version('2.3'):
            if tags is not None:
                extra.update({'tags': tags})
            if system_tags is not None:
                extra.update({'system_tags': system_tags})
        elif tags is not None or system_tags is not None:
            if tags and system_tags:
                system_tags = system_tags[:]
                system_tags += [t for t in tags if t not in system_tags]
            extra.update({'system_tags': system_tags or tags or self.data.system_tags})

        self.send(models.EditRequest(
            model=self.id,
            uri=uri,
            name=name,
            comment=comment,
            labels=labels,
            design=self._wrap_design(design) if design else None,
            framework=framework,
            iteration=iteration,
            **extra
        ))
        self.reload()

    def update_and_upload(self, model_file, design=None, labels=None, name=None, comment=None,
                          tags=None, task_id=None, project_id=None, parent_id=None, framework=None, async_enable=False,
                          target_filename=None, cb=None, iteration=None):
        """ Update the given model for a given task ID """
        if async_enable:
            def callback(uploaded_uri):
                if uploaded_uri is None:
                    return

                # If not successful, mark model as failed_uploading
                if uploaded_uri is False:
                    uploaded_uri = '{}/failed_uploading'.format(self._upload_storage_uri)

                Model._local_model_to_id_uri[str(model_file)] = (self.model_id, uploaded_uri)

                self.update(
                    uri=uploaded_uri,
                    task_id=task_id,
                    name=name,
                    comment=comment,
                    tags=tags,
                    design=design,
                    labels=labels,
                    project_id=project_id,
                    parent_id=parent_id,
                    framework=framework,
                    iteration=iteration,
                )

                if cb:
                    cb(model_file)

            uri = self._upload_model(model_file, async_enable=async_enable, target_filename=target_filename,
                                     cb=callback)
            return uri
        else:
            uri = self._upload_model(model_file, async_enable=async_enable, target_filename=target_filename)
            Model._local_model_to_id_uri[str(model_file)] = (self.model_id, uri)
            self.update(
                uri=uri,
                task_id=task_id,
                name=name,
                comment=comment,
                tags=tags,
                design=design,
                labels=labels,
                project_id=project_id,
                parent_id=parent_id,
                framework=framework,
            )

            return uri

    def update_for_task(self, task_id, name=None, model_id=None, type_="output", iteration=None):
        if Session.check_min_api_version("2.13"):
            req = tasks.AddOrUpdateModelRequest(
                task=task_id, name=name, type=type_, model=model_id, iteration=iteration
            )
        elif type_ == "output":
            # backwards compatibility
            req = models.UpdateForTaskRequest(task=task_id, override_model_id=model_id)
        elif type_ == "input":
            # backwards compatibility, None
            req = None
        else:
            raise ValueError("Type '{}' unsupported (use either 'input' or 'output')".format(type_))

        if req:
            self.send(req)

        self.reload()

    @property
    def model_design(self):
        """ Get the model design. For now, this is stored as a single key in the design dict. """
        try:
            return self._unwrap_design(self.data.design)
        except ValueError:
            # no design is yet specified
            return None

    @property
    def labels(self):
        try:
            return self.data.labels
        except ValueError:
            # no labels is yet specified
            return None

    @property
    def name(self):
        try:
            return self.data.name
        except ValueError:
            # no name is yet specified
            return None

    @property
    def comment(self):
        try:
            return self.data.comment
        except ValueError:
            # no comment is yet specified
            return None

    @property
    def tags(self):
        return self.data.system_tags if hasattr(self.data, 'system_tags') else self.data.tags

    @property
    def task(self):
        try:
            return self.data.task
        except ValueError:
            # no task is yet specified
            return None

    @property
    def uri(self):
        try:
            return self.data.uri
        except ValueError:
            # no uri is yet specified
            return None

    @property
    def locked(self):
        if self.id is None:
            return False
        return bool(self.data.ready)

    def download_model_weights(self, raise_on_error=False):
        """
        Download the model weights into a local file in our cache

        :param bool raise_on_error: If True and the artifact could not be downloaded,
            raise ValueError, otherwise return None on failure and output log warning.

        :return: a local path to a downloaded copy of the model
        """
        uri = self.data.uri
        if not uri or not uri.strip():
            return None

        # check if we already downloaded the file
        downloaded_models = [k for k, (i, u) in Model._local_model_to_id_uri.items() if i == self.id and u == uri]
        for dl_file in downloaded_models:
            if Path(dl_file).exists():
                return dl_file
            # remove non existing model file
            Model._local_model_to_id_uri.pop(dl_file, None)

        local_download = StorageManager.get_local_copy(uri, extract_archive=False)

        # save local model, so we can later query what was the original one
        if local_download is not None:
            Model._local_model_to_id_uri[str(local_download)] = (self.model_id, uri)
        elif raise_on_error:
            raise ValueError("Could not retrieve a local copy of model weights {}, "
                             "failed downloading {}".format(self.model_id, uri))

        return local_download

    @property
    def cache_dir(self):
        return self._cache_dir

    def save_model_design_file(self):
        """ Download model description file into a local file in our cache_dir """
        design = self.model_design
        filename = self.data.name + '.txt'
        p = Path(self.cache_dir) / filename
        # we always write the original model design to file, to prevent any mishaps
        # if p.is_file():
        #     return str(p)
        p.parent.mkdir(parents=True, exist_ok=True)
        p.write_text(six.text_type(design))
        return str(p)

    def get_model_package(self):
        """ Get a named tuple containing the model's weights and design """
        return ModelPackage(weights=self.download_model_weights(), design=self.save_model_design_file())

    def get_model_design(self):
        """ Get model description (text) """
        return self.model_design

    @classmethod
    def get_all(cls, session, log=None, **kwargs):
        req = models.GetAllRequest(**kwargs)
        res = cls._send(session=session, req=req, log=log)
        return res

    def clone(self, name, comment=None, child=True, tags=None, task=None, ready=True):
        """
        Clone this model into a new model.
        :param name: Name for the new model
        :param comment: Optional comment for the new model
        :param child: Should the new model be a child of this model (default True)
        :param tags:  Optional tags for the cloned model
        :param task: Creating Task of the Model
        :param ready:  If True set the true flag for the newly created model
        :return: The new model's ID
        """
        data = self.data
        assert isinstance(data, models.Model)
        parent = self.id if child else None
        extra = {'system_tags': tags or data.system_tags} \
            if Session.check_min_api_version('2.3') else {'tags': tags or data.tags}
        req = models.CreateRequest(
            uri=data.uri,
            name=name,
            labels=data.labels,
            comment=comment or data.comment,
            framework=data.framework,
            design=data.design,
            ready=ready,
            project=data.project,
            parent=parent,
            task=task,
            **extra
        )
        res = self.send(req)
        return res.response.id

    def _create_empty_model(self, upload_storage_uri=None):
        upload_storage_uri = upload_storage_uri or self.upload_storage_uri
        name = make_message('Anonymous model %(time)s')
        uri = '{}/uploading_file'.format(upload_storage_uri or 'file://')
        req = models.CreateRequest(uri=uri, name=name, labels={})
        res = self.send(req)
        if not res:
            return False
        self.id = res.response.id
        return True


# clearml/backend_interface/session.py

from abc import ABCMeta, abstractmethod

import six


class SendError(Exception):
    """ A session send() error class """
    @property
    def result(self):
        return self._result

    def __init__(self, result, *args, **kwargs):
        super(SendError, self).__init__(*args, **kwargs)
        self._result = result


@six.add_metaclass(ABCMeta)
class SessionInterface(object):
    """ Session wrapper interface providing a session property and a send convenience method """

    @property
    @abstractmethod
    def session(self):
        pass

    @abstractmethod
    def send(self, req, ignore_errors=False, raise_on_errors=True, async_enable=False):
        pass


# clearml/backend_interface/setupuploadmixin.py

from abc import abstractproperty

from ..backend_config.bucket_config import S3BucketConfig
from ..storage.helper import StorageHelper


class SetupUploadMixin(object):
    log = abstractproperty()
    storage_uri = abstractproperty()

    def setup_upload(
            self, bucket_name, host=None, access_key=None, secret_key=None, region=None, multipart=True, https=True, verify=True):
        """
        Setup upload options (currently only S3 is supported)

        :param bucket_name: AWS bucket name
        :type bucket_name: str
        :param host: Hostname. Only required in case a Non-AWS S3 solution such as a local Minio server is used)
        :type host: str
        :param access_key: AWS access key. If not provided, we'll attempt to obtain the key from the
            configuration file (bucket-specific, than global)
        :type access_key: str
        :param secret_key: AWS secret key. If not provided, we'll attempt to obtain the secret from the
            configuration file (bucket-specific, than global)
        :type secret_key: str
        :param multipart: Server supports multipart. Only required when using a Non-AWS S3 solution that doesn't support
            multipart.
        :type multipart: bool
        :param https: Server supports HTTPS. Only required when using a Non-AWS S3 solution that only supports HTTPS.
        :type https: bool
        :param region: Bucket region. Required if the bucket doesn't reside in the default region (us-east-1)
        :type region: str
        :param verify: Whether or not to verify SSL certificates. Only required when using a Non-AWS S3 solution that only supports HTTPS with self-signed certificate.
        :type verify: bool
        """
        self._bucket_config = S3BucketConfig(
            bucket=bucket_name,
            host=host,
            key=access_key,
            secret=secret_key,
            multipart=multipart,
            secure=https,
            region=region,
            verify=verify
        )
        self.storage_uri = ('s3://%(host)s/%(bucket_name)s' if host else 's3://%(bucket_name)s') % locals()
        StorageHelper.add_configuration(self._bucket_config, log=self.log)


# clearml/backend_interface/util.py

import getpass
import re
from _socket import gethostname
from datetime import datetime
from typing import Optional

try:
    from datetime import timezone
    utc_timezone = timezone.utc
except ImportError:
    from datetime import tzinfo, timedelta

    class UTC(tzinfo):
        def utcoffset(self, dt):
            return timedelta(0)

        def tzname(self, dt):
            return "UTC"

        def dst(self, dt):
            return timedelta(0)
    utc_timezone = UTC()

from ..backend_api.services import projects, queues
from ..debugging.log import get_logger, LoggerRoot


def make_message(s, **kwargs):
    # noinspection PyBroadException
    try:
        user = getpass.getuser()
    except Exception:
        # noinspection PyBroadException
        try:
            import os
            user = '{}'.format(os.getuid())
        except Exception:
            user = 'unknown'

    # noinspection PyBroadException
    try:
        host = gethostname()
    except Exception:
        host = 'localhost'

    args = dict(
        user=user,
        host=host,
        time=datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
    )
    args.update(kwargs)
    return s % args


def get_existing_project(session, project_name):
    """Return either the project ID if it exists, an empty string if it doesn't or None if backend request failed."""
    res = session.send(projects.GetAllRequest(name=exact_match_regex(project_name), only_fields=['id']))
    if not res:
        return None
    if res.response and res.response.projects:
        return res.response.projects[0].id
    return ""


def get_or_create_project(session, project_name, description=None):
    """Return the ID of an existing project, or if it does not exist, make a new one and return that ID instead."""
    project_id = get_existing_project(session, project_name)
    if project_id:
        return project_id
    if project_id == "":
        # Project was not found, so create a new one
        res = session.send(projects.CreateRequest(name=project_name, description=description or ''))
        return res.response.id

    # This should only happen if backend response was None and so project_id is also None
    return None


def get_queue_id(session, queue):
    # type: ('Session', str) -> Optional[str] # noqa: F821
    if not queue:
        return None

    res = session.send(queues.GetByIdRequest(queue=queue))
    if res and res.response.queue:
        return queue
    res = session.send(queues.GetAllRequest(name=exact_match_regex(queue), only_fields=['id']))
    if res and res.response and res.response.queues:
        if len(res.response.queues) > 1:
            LoggerRoot.get_base_logger().info(
                "Multiple queues with name={}, selecting queue id={}".format(queue, res.response.queues[0].id))
        return res.response.queues[0].id

    return None


# Hack for supporting windows
def get_epoch_beginning_of_time(timezone_info=None):
    return datetime(1970, 1, 1).replace(tzinfo=timezone_info if timezone_info else utc_timezone)


def get_single_result(entity, query, results, log=None, show_results=1, raise_on_error=True, sort_by_date=True):
    if not results:
        if not raise_on_error:
            return None

        raise ValueError('No {entity}s found when searching for `{query}`'.format(**locals()))

    if len(results) > 1:
        if show_results:
            if not log:
                log = get_logger()
            if show_results > 1:
                log.warning('{num} {entity} found when searching for `{query}`'
                            ' (showing first {show_results} {entity}s follow)'.format(num=len(results), **locals()))
            else:
                log.warning('{num} {entity} found when searching for `{query}`'.format(num=len(results), **locals()))

        if sort_by_date:
            relative_time = get_epoch_beginning_of_time()
            # sort results based on timestamp and return the newest one
            if hasattr(results[0], 'last_update'):
                results = sorted(results, key=lambda x: int((x.last_update - relative_time).total_seconds()
                                                            if x.last_update else 0), reverse=True)
            elif hasattr(results[0], 'created'):
                results = sorted(results, key=lambda x: int((x.created - relative_time).total_seconds()
                                                            if x.created else 0), reverse=True)

        if show_results and log:
            for i, obj in enumerate(o if isinstance(o, dict) else o.to_dict() for o in results[:show_results]):
                selected = 'Selected' if i == 0 else 'Additionally found'
                log.warning('{selected} {entity} `{obj[name]}` (id={obj[id]})'.format(**locals()))

        if raise_on_error:
            raise ValueError('More than one {entity}s found when searching for ``{query}`'.format(**locals()))

    return results[0]


def at_least_one(_exception_cls=Exception, _check_none=False, **kwargs):
    actual = [k for k, v in kwargs.items() if (v is not None if _check_none else v)]
    if len(actual) < 1:
        raise _exception_cls('At least one of (%s) is required' % ', '.join(kwargs.keys()))


def mutually_exclusive(_exception_cls=Exception, _require_at_least_one=True, _check_none=False, **kwargs):
    """ Helper for checking mutually exclusive options """
    actual = [k for k, v in kwargs.items() if (v is not None if _check_none else v)]
    if _require_at_least_one:
        at_least_one(_exception_cls=_exception_cls, _check_none=_check_none, **kwargs)
    if len(actual) > 1:
        raise _exception_cls('Only one of (%s) is allowed' % ', '.join(kwargs.keys()))


def validate_dict(obj, key_types, value_types, desc=''):
    if not isinstance(obj, dict):
        raise ValueError('%sexpected a dictionary' % ('%s: ' % desc if desc else ''))
    if not all(isinstance(x, key_types) for x in obj.keys()):
        raise ValueError('%skeys must all be strings' % ('%s ' % desc if desc else ''))
    if not all(isinstance(x, value_types) for x in obj.values()):
        raise ValueError('%svalues must all be integers' % ('%s ' % desc if desc else ''))


def exact_match_regex(name):
    """ Convert string to a regex representing an exact match """
    return '^%s$' % re.escape(name or '')


def datetime_to_isoformat(o):
    if isinstance(o, datetime):
        return o.isoformat()
    return None


def datetime_from_isoformat(o):
    # type: (str) -> Optional[datetime]
    if not o:
        return None
    if isinstance(o, datetime):
        return o
    try:
        return datetime.strptime(o.split('+')[0], "%Y-%m-%dT%H:%M:%S.%f")
    except ValueError:
        return datetime.strptime(o.split('+')[0], "%Y-%m-%dT%H:%M:%S")


# clearml/config/__init__.py

""" Configuration module. Uses backend_config to load system configuration. """
import logging
import os
import sys
from os.path import expandvars, expanduser

from ..backend_api import load_config
from ..backend_config import Config
from ..backend_config.bucket_config import S3BucketConfigurations

from .defs import *  # noqa: F403
from .remote import running_remotely_task_id as _running_remotely_task_id

# config_obj = load_config(Path(__file__).parent)  # noqa: F405
# config_obj.initialize_logging()
# config = config_obj.get("sdk")
from ..utilities.proxy_object import LazyEvalWrapper


class ConfigWrapper(object):
    _config = None

    @classmethod
    def _init(cls):
        if cls._config is None:
            cls._config = load_config(Path(__file__).parent)  # noqa: F405
            cls._config.initialize_logging()

    @classmethod
    def get(cls, *args, **kwargs):
        cls._init()
        return cls._config.get(*args, **kwargs)

    @classmethod
    def set_overrides(cls, *args, **kwargs):
        cls._init()
        return cls._config.set_overrides(*args, **kwargs)


class ConfigSDKWrapper(object):
    _config_sdk = None

    @classmethod
    def _init(cls):
        if cls._config_sdk is None:
            cls._config_sdk = ConfigWrapper.get("sdk")

    @classmethod
    def get(cls, *args, **kwargs):
        cls._init()
        return cls._config_sdk.get(*args, **kwargs)

    @classmethod
    def set_overrides(cls, *args, **kwargs):
        cls._init()
        return cls._config_sdk.set_overrides(*args, **kwargs)


def deferred_config(key=None, default=Config._MISSING, transform=None, multi=None):
    return LazyEvalWrapper(
        callback=lambda:
        (ConfigSDKWrapper.get(key, default) if not multi else
         next((ConfigSDKWrapper.get(*a) for a in multi if ConfigSDKWrapper.get(*a)), None))
        if transform is None
        else (transform() if key is None else transform(ConfigSDKWrapper.get(key, default) if not multi else  # noqa
                                                        next((ConfigSDKWrapper.get(*a) for a in multi
                                                              if ConfigSDKWrapper.get(*a)), None)))
    )


config_obj = ConfigWrapper
config = ConfigSDKWrapper

""" Configuration object reflecting the merged SDK section of all available configuration files """


def get_cache_dir():
    cache_base_dir = Path(  # noqa: F405
        expandvars(
            expanduser(
                CLEARML_CACHE_DIR.get() or  # noqa: F405
                config.get("storage.cache.default_base_dir") or
                DEFAULT_CACHE_DIR  # noqa: F405
            )
        )
    )
    return cache_base_dir


def get_offline_dir(task_id=None):
    if not task_id:
        return get_cache_dir() / 'offline'
    return get_cache_dir() / 'offline' / task_id


def get_config_for_bucket(base_url, extra_configurations=None):
    config_list = S3BucketConfigurations.from_config(config.get("aws.s3"))

    for configuration in extra_configurations or []:
        config_list.add_config(configuration)

    return config_list.get_config_by_uri(base_url)


def get_remote_task_id():
    return _running_remotely_task_id


def running_remotely():
    return bool(_running_remotely_task_id)


def get_log_to_backend(default=None):
    return LOG_TO_BACKEND_ENV_VAR.get(default=default)  # noqa: F405


def get_node_count():
    # noinspection PyBroadException
    try:
        mpi_world_rank = int(os.environ.get('OMPI_COMM_WORLD_NODE_RANK', os.environ.get('PMI_RANK')))
        return mpi_world_rank
    except Exception:
        pass

    # noinspection PyBroadException
    try:
        mpi_rank = int(os.environ.get('OMPI_COMM_WORLD_RANK', os.environ.get('SLURM_JOB_NUM_NODES')))
        return mpi_rank
    except Exception:
        pass

    # check if we have pyTorch node/worker ID (only if torch was already imported)
    if 'torch' in sys.modules:
        # noinspection PyBroadException
        try:
            from torch.utils.data.dataloader import get_worker_info  # noqa
            worker_info = get_worker_info()
            if worker_info:
                return int(worker_info.num_workers)
        except Exception:
            pass

    return None


def get_node_id(default=0):
    node_id = NODE_ID_ENV_VAR.get()  # noqa: F405

    # noinspection PyBroadException
    try:
        mpi_world_rank = int(os.environ.get('OMPI_COMM_WORLD_NODE_RANK', os.environ.get('PMI_RANK')))
    except Exception:
        mpi_world_rank = None

    # noinspection PyBroadException
    try:
        mpi_rank = int(os.environ.get(
            'OMPI_COMM_WORLD_RANK', os.environ.get('SLURM_PROCID', os.environ.get('SLURM_NODEID')))
        )
    except Exception:
        mpi_rank = None

    # if we have no node_id, use the mpi rank
    if node_id is None and (mpi_world_rank is not None or mpi_rank is not None):
        node_id = mpi_world_rank if mpi_world_rank is not None else mpi_rank

    # if node is is till None, use the default
    if node_id is None:
        node_id = default

    torch_rank = None
    # check if we have pyTorch node/worker ID (only if torch was already imported)
    if 'torch' in sys.modules:
        # noinspection PyBroadException
        try:
            from torch.utils.data.dataloader import get_worker_info  # noqa
            worker_info = get_worker_info()
            if not worker_info:
                torch_rank = None
            else:
                w_id = worker_info.id
                # noinspection PyBroadException
                try:
                    torch_rank = int(w_id)
                except Exception:
                    # guess a number based on wid hopefully unique value
                    import hashlib
                    h = hashlib.md5()
                    h.update(str(w_id).encode('utf-8'))
                    torch_rank = int(h.hexdigest(), 16)
        except Exception:
            torch_rank = None

    # if we also have a torch rank add it to the node rank
    if torch_rank is not None:
        # Since we dont know the world rank, we assume it is not bigger than 10k
        node_id = (10000 * node_id) + torch_rank

    return node_id


def get_is_master_node():
    global __force_master_node
    if __force_master_node:
        return True

    return get_node_id(default=0) == 0


def get_log_redirect_level():
    """ Returns which log level (and up) should be redirected to stderr. None means no redirection. """
    value = LOG_STDERR_REDIRECT_LEVEL.get()  # noqa: F405
    try:
        if value:
            return logging._checkLevel(value)  # noqa
    except (ValueError, TypeError, AttributeError):
        pass


def dev_worker_name():
    return DEV_WORKER_NAME.get()  # noqa: F405


def __set_is_master_node():
    # noinspection PyBroadException
    try:
        # pop both set the first
        env_a = os.environ.pop('CLEARML_FORCE_MASTER_NODE', None)
        env_b = os.environ.pop('TRAINS_FORCE_MASTER_NODE', None)
        force_master_node = env_a or env_b
    except Exception:
        force_master_node = None

    if force_master_node is not None:
        # noinspection PyBroadException
        try:
            force_master_node = bool(int(force_master_node))
        except Exception:
            force_master_node = None

    return force_master_node


__force_master_node = __set_is_master_node()


# clearml/config/cache.py

import json
from os.path import expanduser

import six
from pathlib2 import Path

from . import running_remotely
from .defs import SESSION_CACHE_FILE


class SessionCache(object):
    """
    Handle SDK session cache.
    TODO: Improve error handling to something like "except (FileNotFoundError, PermissionError, JSONDecodeError)"
    TODO: that's both six-compatible and tested
    """

    SESSION_CACHE_FOLDER = "~/.clearml"

    @classmethod
    def _load_cache(cls):
        # noinspection PyBroadException
        try:
            flag = 'rb' if six.PY2 else 'rt'
            with (Path(expanduser(cls.SESSION_CACHE_FOLDER)) / SESSION_CACHE_FILE).open(flag) as fp:
                return json.load(fp)
        except Exception:
            return {}

    @classmethod
    def _store_cache(cls, cache):
        # noinspection PyBroadException
        try:
            Path(expanduser(cls.SESSION_CACHE_FOLDER)).mkdir(parents=True, exist_ok=True)
            flag = 'wb' if six.PY2 else 'wt'
            with (Path(expanduser(cls.SESSION_CACHE_FOLDER)) / SESSION_CACHE_FILE).open(flag) as fp:
                json.dump(cache, fp)
        except Exception:
            pass

    @classmethod
    def store_dict(cls, unique_cache_name, dict_object):
        # type: (str, dict) -> None
        # disable session cache when running in remote execution mode
        if running_remotely():
            return
        cache = cls._load_cache()
        cache[unique_cache_name] = dict_object
        cls._store_cache(cache)

    @classmethod
    def load_dict(cls, unique_cache_name):
        # type: (str) -> dict
        # disable session cache when running in remote execution mode
        if running_remotely():
            return {}
        cache = cls._load_cache()
        return cache.get(unique_cache_name, {}) if cache else {}


# clearml/config/defs.py

import tempfile

from ..backend_config import EnvEntry
from ..backend_config.converters import base64_to_text, or_
from pathlib2 import Path

SESSION_CACHE_FILE = ".session.json"
DEFAULT_CACHE_DIR = str(Path(tempfile.gettempdir()) / "clearml_cache")

TASK_ID_ENV_VAR = EnvEntry("CLEARML_TASK_ID", "TRAINS_TASK_ID")
DOCKER_IMAGE_ENV_VAR = EnvEntry("CLEARML_DOCKER_IMAGE", "TRAINS_DOCKER_IMAGE")
DOCKER_BASH_SETUP_ENV_VAR = EnvEntry("CLEARML_DOCKER_BASH_SCRIPT")
LOG_TO_BACKEND_ENV_VAR = EnvEntry("CLEARML_LOG_TASK_TO_BACKEND", "TRAINS_LOG_TASK_TO_BACKEND", type=bool)
NODE_ID_ENV_VAR = EnvEntry("CLEARML_NODE_ID", "TRAINS_NODE_ID", type=int)
PROC_MASTER_ID_ENV_VAR = EnvEntry("CLEARML_PROC_MASTER_ID", "TRAINS_PROC_MASTER_ID", type=str)
LOG_STDERR_REDIRECT_LEVEL = EnvEntry("CLEARML_LOG_STDERR_REDIRECT_LEVEL", "TRAINS_LOG_STDERR_REDIRECT_LEVEL")
DEV_WORKER_NAME = EnvEntry("CLEARML_WORKER_NAME", "TRAINS_WORKER_NAME")
DEV_TASK_NO_REUSE = EnvEntry("CLEARML_TASK_NO_REUSE", "TRAINS_TASK_NO_REUSE", type=bool)
TASK_LOG_ENVIRONMENT = EnvEntry("CLEARML_LOG_ENVIRONMENT", "TRAINS_LOG_ENVIRONMENT", type=str)
CLEARML_CACHE_DIR = EnvEntry("CLEARML_CACHE_DIR", "TRAINS_CACHE_DIR")
DEBUG_SIMULATE_REMOTE_TASK = EnvEntry("CLEARML_SIMULATE_REMOTE_TASK", type=bool)
DEV_DEFAULT_OUTPUT_URI = EnvEntry("CLEARML_DEFAULT_OUTPUT_URI", type=str)
TASK_SET_ITERATION_OFFSET = EnvEntry("CLEARML_SET_ITERATION_OFFSET", type=int)

LOG_LEVEL_ENV_VAR = EnvEntry("CLEARML_LOG_LEVEL", "TRAINS_LOG_LEVEL", converter=or_(int, str))

SUPPRESS_UPDATE_MESSAGE_ENV_VAR = EnvEntry("CLEARML_SUPPRESS_UPDATE_MESSAGE", "TRAINS_SUPPRESS_UPDATE_MESSAGE", type=bool)

# Repository detection
VCS_REPO_TYPE = EnvEntry("CLEARML_VCS_REPO_TYPE", "TRAINS_VCS_REPO_TYPE", default="git")
VCS_REPOSITORY_URL = EnvEntry("CLEARML_VCS_REPO_URL", "TRAINS_VCS_REPO_URL")
VCS_COMMIT_ID = EnvEntry("CLEARML_VCS_COMMIT_ID", "TRAINS_VCS_COMMIT_ID")
VCS_BRANCH = EnvEntry("CLEARML_VCS_BRANCH", "TRAINS_VCS_BRANCH")
VCS_ROOT = EnvEntry("CLEARML_VCS_ROOT", "TRAINS_VCS_ROOT")
VCS_WORK_DIR = EnvEntry("CLEARML_VCS_WORK_DIR")
VCS_STATUS = EnvEntry("CLEARML_VCS_STATUS", "TRAINS_VCS_STATUS", converter=base64_to_text)
VCS_DIFF = EnvEntry("CLEARML_VCS_DIFF", "TRAINS_VCS_DIFF", converter=base64_to_text)


# clearml/config/remote.py

from .defs import TASK_ID_ENV_VAR

running_remotely_task_id = TASK_ID_ENV_VAR.get()


def override_current_task_id(task_id):
    """
    Overrides the current task id to simulate remote running with a specific task.

    Use for testing and debug only.

    :param task_id: The task's id to use as the remote task.
        Pass None to simulate local execution.
    """

    global running_remotely_task_id
    running_remotely_task_id = task_id
    # make sure we change the cached value as well.
    import clearml
    clearml.config._running_remotely_task_id = task_id


# clearml/datasets/__init__.py

from .dataset import FileEntry, Dataset

__all__ = [
  "FileEntry",
  "Dataset",
]


# clearml/datasets/dataset.py

import json
import os
import shutil
from copy import deepcopy, copy
from fnmatch import fnmatch
from multiprocessing import cpu_count
from multiprocessing.pool import ThreadPool
from tempfile import mkstemp, mkdtemp
from typing import Union, Optional, Sequence, List, Dict, Any, Mapping
from zipfile import ZipFile, ZIP_DEFLATED

from attr import attrs, attrib
from pathlib2 import Path

from .. import Task, StorageManager, Logger
from ..backend_api.session.client import APIClient
from ..backend_interface.task.development.worker import DevWorker
from ..backend_interface.util import mutually_exclusive, exact_match_regex, get_existing_project
from ..config import deferred_config
from ..debugging.log import LoggerRoot
from ..storage.helper import StorageHelper
from ..storage.cache import CacheManager
from ..storage.util import sha256sum, is_windows, md5text, format_size

try:
    from pathlib import Path as _Path  # noqa
except ImportError:
    _Path = None


@attrs
class FileEntry(object):
    relative_path = attrib(default=None, type=str)
    hash = attrib(default=None, type=str)
    parent_dataset_id = attrib(default=None, type=str)
    size = attrib(default=None, type=int)
    # support multi part artifact storage
    artifact_name = attrib(default=None, type=str)
    # cleared when file is uploaded.
    local_path = attrib(default=None, type=str)

    def as_dict(self):
        # type: () -> Dict
        state = dict(relative_path=self.relative_path, hash=self.hash,
                     parent_dataset_id=self.parent_dataset_id, size=self.size,
                     artifact_name=self.artifact_name,
                     **dict([('local_path', self.local_path)] if self.local_path else ()))
        return state


class Dataset(object):
    __private_magic = 42 * 1337
    __state_entry_name = 'state'
    __default_data_entry_name = 'data'
    __data_entry_name_prefix = 'data_'
    __cache_context = 'datasets'
    __tag = 'dataset'
    __cache_folder_prefix = 'ds_'
    __dataset_folder_template = CacheManager.set_context_folder_lookup(__cache_context, "{0}_archive_{1}")
    __preview_max_file_entries = 15000
    __preview_max_size = 5 * 1024 * 1024
    _dataset_chunk_size_mb = deferred_config("storage.dataset_chunk_size_mb", 512, transform=int)

    def __init__(self, _private, task=None, dataset_project=None, dataset_name=None, dataset_tags=None):
        # type: (int, Optional[Task], Optional[str], Optional[str], Optional[Sequence[str]]) -> ()
        """
        Do not use directly! Use Dataset.create(...) or Dataset.get(...) instead.
        """
        assert _private == self.__private_magic
        # key for the dataset file entries are the relative path within the data
        self._dataset_file_entries = {}  # type: Dict[str, FileEntry]
        # this will create a graph of all the dependencies we have, each entry lists it's own direct parents
        self._dependency_graph = {}  # type: Dict[str, List[str]]
        if task:
            self._task_pinger = None
            self._created_task = False
            task_status = task.data.status
            # if we are continuing aborted Task, force the state
            if str(task_status) == 'stopped':
                # print warning that we are opening a stopped dataset:
                LoggerRoot.get_base_logger().warning(
                    'Reopening aborted Dataset, any change will clear and overwrite current state')
                task.mark_started(force=True)
                task_status = 'in_progress'

            # If we are reusing the main current Task, make sure we set its type to data_processing
            if str(task_status) in ('created', 'in_progress'):
                if str(task.task_type) != str(Task.TaskTypes.data_processing):
                    task.set_task_type(task_type=Task.TaskTypes.data_processing)
                task_system_tags = task.get_system_tags() or []
                if self.__tag not in task_system_tags:
                    task.set_system_tags(task_system_tags + [self.__tag])
                if dataset_tags:
                    task.set_tags((task.get_tags() or []) + list(dataset_tags))
        else:
            self._created_task = True
            task = Task.create(
                project_name=dataset_project, task_name=dataset_name, task_type=Task.TaskTypes.data_processing)
            # set default output_uri
            task.output_uri = True
            task.set_system_tags((task.get_system_tags() or []) + [self.__tag])
            if dataset_tags:
                task.set_tags((task.get_tags() or []) + list(dataset_tags))
            task.mark_started()
            # generate the script section
            script = \
                'from clearml import Dataset\n\n' \
                'ds = Dataset.create(dataset_project=\'{dataset_project}\', dataset_name=\'{dataset_name}\')\n'.format(
                    dataset_project=dataset_project, dataset_name=dataset_name)
            task.data.script.diff = script
            task.data.script.working_dir = '.'
            task.data.script.entry_point = 'register_dataset.py'
            from clearml import __version__
            task.data.script.requirements = {'pip': 'clearml == {}\n'.format(__version__)}
            # noinspection PyProtectedMember
            task._edit(script=task.data.script)

            # if the task is running make sure we ping to the server so it will not be aborted by a watchdog
            self._task_pinger = DevWorker()
            self._task_pinger.register(task, stop_signal_support=False)
            # set the newly created Dataset parent ot the current Task, so we know who created it.
            if Task.current_task() and Task.current_task().id != task.id:
                task.set_parent(Task.current_task())

        # store current dataset Task
        self._task = task
        # store current dataset id
        self._id = task.id
        # store the folder where the dataset was downloaded to
        self._local_base_folder = None  # type: Optional[Path]
        # dirty flag, set True by any function call changing the dataset (regardless of weather it did anything)
        self._dirty = False
        self._using_current_task = False
        # set current artifact name to be used (support for multiple upload sessions)
        self._data_artifact_name = self._get_next_data_artifact_name()
        # store a cached lookup of the number of chunks each parent dataset has.
        # this will help with verifying we have n up-to-date partial local copy
        self._dependency_chunk_lookup = None  # type: Optional[Dict[str, int]]

    @property
    def id(self):
        # type: () -> str
        return self._id

    @property
    def file_entries(self):
        # type: () -> List[FileEntry]
        return list(self._dataset_file_entries.values())

    @property
    def file_entries_dict(self):
        # type: () -> Mapping[str, FileEntry]
        """
        Notice this call returns an internal representation, do not modify!
        :return: dict with relative file path as key, and FileEntry as value
        """
        return self._dataset_file_entries

    @property
    def project(self):
        # type: () -> str
        return self._task.get_project_name()

    @property
    def name(self):
        # type: () -> str
        return self._task.name

    @property
    def tags(self):
        # type: () -> List[str]
        return self._task.get_tags() or []

    @tags.setter
    def tags(self, values):
        # type: (List[str]) -> ()
        self._task.set_tags(values or [])

    def add_files(
            self,
            path,  # type: Union[str, Path, _Path]
            wildcard=None,  # type: Optional[Union[str, Sequence[str]]]
            local_base_folder=None,  # type: Optional[str]
            dataset_path=None,  # type: Optional[str]
            recursive=True,  # type: bool
            verbose=False  # type: bool
    ):
        # type: (...) -> ()
        """
        Add a folder into the current dataset. calculate file hash,
        and compare against parent, mark files to be uploaded

        :param path: Add a folder/file to the dataset
        :param wildcard: add only specific set of files.
            Wildcard matching, can be a single string or a list of wildcards)
        :param local_base_folder: files will be located based on their relative path from local_base_folder
        :param dataset_path: where in the dataset the folder/files should be located
        :param recursive: If True match all wildcard files recursively
        :param verbose: If True print to console files added/modified
        :return: number of files added
        """
        self._dirty = True
        self._task.get_logger().report_text(
            'Adding files to dataset: {}'.format(
                dict(path=path, wildcard=wildcard, local_base_folder=local_base_folder,
                     dataset_path=dataset_path, recursive=recursive, verbose=verbose)),
            print_console=False)

        num_added = self._add_files(
            path=path, wildcard=wildcard, local_base_folder=local_base_folder,
            dataset_path=dataset_path, recursive=recursive, verbose=verbose)

        # update the task script
        self._add_script_call(
            'add_files', path=path, wildcard=wildcard, local_base_folder=local_base_folder,
            dataset_path=dataset_path, recursive=recursive)

        self._serialize()

        return num_added

    def remove_files(self, dataset_path=None, recursive=True, verbose=False):
        # type: (Optional[str], bool, bool) -> int
        """
        Remove files from the current dataset

        :param dataset_path: Remove files from the dataset.
            The path is always relative to the dataset (e.g 'folder/file.bin')
        :param recursive: If True match all wildcard files recursively
        :param verbose: If True print to console files removed
        :return: Number of files removed
        """
        self._task.get_logger().report_text(
            'Removing files from dataset: {}'.format(
                dict(dataset_path=dataset_path, recursive=recursive, verbose=verbose)),
            print_console=False)

        if dataset_path and dataset_path.startswith('/'):
            dataset_path = dataset_path[1:]

        num_files = len(self._dataset_file_entries)
        org_files = list(self._dataset_file_entries.keys()) if verbose else None

        if not recursive:
            self._dataset_file_entries = {
                k: v for k, v in self._dataset_file_entries.items()
                if not fnmatch(k + '/', dataset_path + '/')}
        else:
            wildcard = dataset_path.split('/')[-1]
            path = dataset_path[:-len(dataset_path)] + '*'

            self._dataset_file_entries = {
                k: v for k, v in self._dataset_file_entries.items()
                if not (fnmatch(k, path) and fnmatch(k if '/' in k else '/{}'.format(k), '*/' + wildcard))}

        if verbose and org_files:
            for f in org_files:
                if f not in self._dataset_file_entries:
                    self._task.get_logger().report_text('Remove {}'.format(f))

        # update the task script
        self._add_script_call(
            'remove_files', dataset_path=dataset_path, recursive=recursive)

        self._serialize()

        return num_files - len(self._dataset_file_entries)

    def sync_folder(self, local_path, dataset_path=None, verbose=False):
        # type: (Union[Path, _Path, str], Union[Path, _Path, str], bool) -> (int, int)
        """
        Synchronize the dataset with a local folder. The dataset is synchronized from the
            relative_base_folder (default: dataset root)  and deeper with the specified local path.

        :param local_path: Local folder to sync (assumes all files and recursive)
        :param dataset_path: Target dataset path to sync with (default the root of the dataset)
        :param verbose: If true print to console files added/modified/removed
        :return: number of files removed, number of files modified/added
        """
        def filter_f(f):
            keep = (not f.relative_path.startswith(relative_prefix) or
                    (local_path / f.relative_path[len(relative_prefix):]).is_file())
            if not keep and verbose:
                self._task.get_logger().report_text('Remove {}'.format(f.relative_path))
            return keep

        self._task.get_logger().report_text(
            'Syncing local copy with dataset: {}'.format(
                dict(local_path=local_path, dataset_path=dataset_path, verbose=verbose)),
            print_console=False)

        self._dirty = True
        local_path = Path(local_path)

        # Path().as_posix() will never end with /
        relative_prefix = (Path(dataset_path).as_posix() + '/') if dataset_path else ''

        # remove files
        num_files = len(self._dataset_file_entries)
        self._dataset_file_entries = {
            k: f for k, f in self._dataset_file_entries.items() if filter_f(f)}
        removed_files = num_files - len(self._dataset_file_entries)

        # add remaining files
        added_files = self._add_files(path=local_path, dataset_path=dataset_path, recursive=True, verbose=verbose)

        if verbose:
            self._task.get_logger().report_text(
                'Syncing folder {} : {} files removed, {} added / modified'.format(
                    local_path.as_posix(), removed_files, added_files))

        # update the task script
        self._add_script_call(
            'sync_folder', local_path=local_path, dataset_path=dataset_path)

        self._serialize()
        return removed_files, added_files

    def upload(self, show_progress=True, verbose=False, output_url=None, compression=None, chunk_size=None):
        # type: (bool, bool, Optional[str], Optional[str], int) -> ()
        """
        Start file uploading, the function returns when all files are uploaded.

        :param show_progress: If True show upload progress bar
        :param verbose: If True print verbose progress report
        :param output_url: Target storage for the compressed dataset (default: file server)
            Examples: `s3://bucket/data`, `gs://bucket/data` , `azure://bucket/data` , `/mnt/share/data`
        :param compression: Compression algorithm for the Zipped dataset file (default: ZIP_DEFLATED)
        :param chunk_size: Artifact chunk size (MB) for the compressed dataset,
            if not provided (None) use the default chunk size (512mb).
            If -1 is provided, use a single zip artifact for the entire dataset change-set (old behaviour)
        """
        # set output_url
        if output_url:
            self._task.output_uri = output_url

        self._task.get_logger().report_text(
            'Uploading dataset files: {}'.format(
                dict(show_progress=show_progress, verbose=verbose, output_url=output_url, compression=compression)),
            print_console=False)

        list_zipped_artifacts = []  # List[Tuple[Path, int, str, str]]
        list_file_entries = list(self._dataset_file_entries.values())
        total_size = 0
        chunk_size = int(self._dataset_chunk_size_mb if not chunk_size else chunk_size)
        try:
            from tqdm import tqdm  # noqa
            a_tqdm = tqdm(total=len(list_file_entries))
        except ImportError:
            a_tqdm = None

        while list_file_entries:
            fd, zip_file = mkstemp(
                prefix='dataset.{}.'.format(self._id), suffix='.zip'
            )
            archive_preview = ''
            count = 0
            processed = 0
            zip_file = Path(zip_file)
            print('{}Compressing local files, chunk {} [remaining {} files]'.format(
                '\n' if a_tqdm else '', 1+len(list_zipped_artifacts), len(list_file_entries)))
            try:
                with ZipFile(zip_file.as_posix(), 'w', allowZip64=True, compression=compression or ZIP_DEFLATED) as zf:
                    for file_entry in list_file_entries:
                        processed += 1
                        if a_tqdm:
                            a_tqdm.update()
                        if not file_entry.local_path:
                            # file is already in an uploaded artifact
                            continue
                        filename = Path(file_entry.local_path)
                        if not filename.is_file():
                            LoggerRoot.get_base_logger().warning(
                                "Could not store dataset file {}. File skipped".format(file_entry.local_path))
                            # mark for removal
                            file_entry.relative_path = None
                            continue
                        if verbose:
                            self._task.get_logger().report_text('Compressing {}'.format(filename.as_posix()))

                        relative_file_name = file_entry.relative_path
                        zf.write(filename.as_posix(), arcname=relative_file_name)
                        archive_preview += '{} - {}\n'.format(
                            relative_file_name, format_size(filename.stat().st_size))
                        file_entry.artifact_name = self._data_artifact_name
                        count += 1

                        # limit the size of a single artifact
                        if chunk_size > 0 and zip_file.stat().st_size >= chunk_size * (1024**2):
                            break
            except Exception as e:
                # failed uploading folder:
                LoggerRoot.get_base_logger().warning(
                    'Exception {}\nFailed zipping dataset.'.format(e))
                return False
            finally:
                os.close(fd)

            if not count:
                zip_file.unlink()

            total_size += zip_file.stat().st_size
            # let's see what's left
            list_file_entries = list_file_entries[processed:]
            # update the artifact preview
            archive_preview = 'Dataset archive content [{} files]:\n'.format(count) + archive_preview
            # add into the list
            list_zipped_artifacts += [(zip_file, count, archive_preview, self._data_artifact_name)]
            # next artifact name to use
            self._data_artifact_name = self._get_next_data_artifact_name(self._data_artifact_name)

        if a_tqdm:
            a_tqdm.close()

        self._task.get_logger().report_text(
            'File compression completed: total size {}, {} chunked stored (average size {})'.format(
                format_size(total_size),
                len(list_zipped_artifacts),
                format_size(total_size / len(list_zipped_artifacts))))

        if not list_zipped_artifacts:
            LoggerRoot.get_base_logger().info('No pending files, skipping upload.')
            self._dirty = False
            self._serialize()
            return True

        for i, (zip_file, count, archive_preview, artifact_name) in enumerate(list_zipped_artifacts):
            # noinspection PyBroadException
            try:
                # let's try to rename it
                new_zip_file = zip_file.parent / 'dataset.{}.zip'.format(self._id)
                zip_file.rename(new_zip_file)
                zip_file = new_zip_file
            except Exception:
                pass

            # start upload
            zip_file_size = format_size(Path(zip_file).stat().st_size)
            self._task.get_logger().report_text(
                'Uploading compressed dataset changes {}/{} ({} files {}) to {}'.format(
                    i+1, len(list_zipped_artifacts), count, zip_file_size, self.get_default_storage()))

            self._task.upload_artifact(
                name=artifact_name, artifact_object=Path(zip_file), preview=archive_preview,
                delete_after_upload=True, wait_on_upload=True)

            # mark as upload completed and serialize
            for file_entry in self._dataset_file_entries.values():
                if file_entry.parent_dataset_id == self._id and file_entry.artifact_name == artifact_name:
                    file_entry.local_path = None
            # serialize current state
            self._serialize()

        # remove files that could not be zipped, containing Null relative Path
        self._dataset_file_entries = {
            k: v for k, v in self._dataset_file_entries.items() if v.relative_path is not None}
        # report upload completed
        self._task.get_logger().report_text('Upload completed ({})'.format(format_size(total_size)))

        self._add_script_call(
            'upload', show_progress=show_progress, verbose=verbose, output_url=output_url, compression=compression)

        self._dirty = False
        self._serialize()

    def finalize(self, verbose=False, raise_on_error=True, auto_upload=False):
        # type: (bool, bool, bool) -> bool
        """
        Finalize the dataset publish dataset Task. upload must first called to verify there are not pending uploads.
        If files do need to be uploaded, it throws an exception (or return False)

        :param verbose: If True print verbose progress report
        :param raise_on_error: If True raise exception if dataset finalizing failed
        :param auto_upload: Automatically upload dataset if not called yet, will upload to default location.
        """
        # check we do not have files waiting for upload.
        if self._dirty:
            if auto_upload:
                self._task.get_logger().report_text("Pending uploads, starting dataset upload to {}"
                                                    .format(self.get_default_storage()))
                self.upload()
            elif raise_on_error:
                raise ValueError("Cannot finalize dataset, pending uploads. Call Dataset.upload(...)")
            else:
                return False

        status = self._task.get_status()
        if status not in ('in_progress', 'created'):
            raise ValueError("Cannot finalize dataset, status '{}' is not valid".format(status))

        self._task.get_logger().report_text('Finalizing dataset', print_console=False)

        # make sure we have no redundant parent versions
        self._serialize(update_dependency_chunk_lookup=True)
        self._add_script_call('finalize')
        if verbose:
            print('Updating statistics and genealogy')
        self._report_dataset_genealogy()
        hashed_nodes = [self._get_dataset_id_hash(k) for k in self._dependency_graph.keys()]
        self._task.comment = 'Dependencies: {}\n'.format(hashed_nodes)
        if self._using_current_task:
            self._task.flush(wait_for_uploads=True)
        else:
            self._task.close()
            self._task.mark_completed()

        if self._task_pinger:
            self._task_pinger.unregister()
            self._task_pinger = None

        return True

    def publish(self, raise_on_error=True):
        # type: (bool) -> bool
        """
        Publish the dataset
        If dataset is not finalize, throw exception

        :param raise_on_error: If True raise exception if dataset publishing failed
        """
        # check we can publish this dataset
        if not self.is_final():
            raise ValueError("Cannot publish dataset, dataset in status {}.".format(self._task.get_status()))

        self._task.publish(ignore_errors=raise_on_error)
        return True

    def is_final(self):
        # type: () -> bool
        """
        Return True if the dataset was finalized and cannot be changed any more.

        :return: True if dataset if final
        """
        return self._task.get_status() not in (
            Task.TaskStatusEnum.in_progress, Task.TaskStatusEnum.created, Task.TaskStatusEnum.failed)

    def get_local_copy(self, use_soft_links=None, part=None, num_parts=None, raise_on_error=True):
        # type: (bool, Optional[int], Optional[int], bool) -> str
        """
        return a base folder with a read-only (immutable) local copy of the entire dataset
            download and copy / soft-link, files from all the parent dataset versions

        :param use_soft_links: If True use soft links, default False on windows True on Posix systems
        :param part: Optional, if provided only download the selected part (index) of the Dataset.
            First part number is `0` and last part is `num_parts-1`
            Notice, if `num_parts` is not provided, number of parts will be equal to the total number of chunks
            (i.e. sum over all chunks from the specified Dataset including all parent Datasets).
            This argument is passed to parent datasets, as well as the implicit `num_parts`,
            allowing users to get a partial copy of the entire dataset, for multi node/step processing.
        :param num_parts: Optional, If specified normalize the number of chunks stored to the
            requested number of parts. Notice that the actual chunks used per part are rounded down.
            Example: Assuming total 8 chunks for this dataset (including parent datasets),
            and `num_parts=5`, the chunk index used per parts would be:
            part=0 -> chunks[0,5], part=1 -> chunks[1,6], part=2 -> chunks[2,7], part=3 -> chunks[3, ]
        :param raise_on_error: If True raise exception if dataset merging failed on any file

        :return: A base folder for the entire dataset
        """
        assert self._id
        if not self._task:
            self._task = Task.get_task(task_id=self._id)

        # now let's merge the parents
        target_folder = self._merge_datasets(
            use_soft_links=use_soft_links, raise_on_error=raise_on_error, part=part, num_parts=num_parts)
        return target_folder

    def get_mutable_local_copy(self, target_folder, overwrite=False, part=None, num_parts=None, raise_on_error=True):
        # type: (Union[Path, _Path, str], bool, Optional[int], Optional[int], bool) -> Optional[str]
        """
        return a base folder with a writable (mutable) local copy of the entire dataset
            download and copy / soft-link, files from all the parent dataset versions

        :param target_folder: Target folder for the writable copy
        :param overwrite: If True, recursively delete the target folder before creating a copy.
            If False (default) and target folder contains files, raise exception or return None
        :param part: Optional, if provided only download the selected part (index) of the Dataset.
            First part number is `0` and last part is `num_parts-1`
            Notice, if `num_parts` is not provided, number of parts will be equal to the total number of chunks
            (i.e. sum over all chunks from the specified Dataset including all parent Datasets).
            This argument is passed to parent datasets, as well as the implicit `num_parts`,
            allowing users to get a partial copy of the entire dataset, for multi node/step processing.
        :param num_parts: Optional, If specified normalize the number of chunks stored to the
            requested number of parts. Notice that the actual chunks used per part are rounded down.
            Example: Assuming total 8 chunks for this dataset (including parent datasets),
            and `num_parts=5`, the chunk index used per parts would be:
            part=0 -> chunks[0,5], part=1 -> chunks[1,6], part=2 -> chunks[2,7], part=3 -> chunks[3, ]
        :param raise_on_error: If True raise exception if dataset merging failed on any file

        :return: A the target folder containing the entire dataset
        """
        assert self._id
        target_folder = Path(target_folder).absolute()
        target_folder.mkdir(parents=True, exist_ok=True)
        # noinspection PyBroadException
        try:
            target_folder.rmdir()
        except Exception:
            if not overwrite:
                if raise_on_error:
                    raise ValueError("Target folder {} already contains files".format(target_folder.as_posix()))
                else:
                    return None
            shutil.rmtree(target_folder.as_posix())

        ro_folder = self.get_local_copy(part=part, num_parts=num_parts, raise_on_error=raise_on_error)
        shutil.copytree(ro_folder, target_folder.as_posix(), symlinks=False)
        return target_folder.as_posix()

    def list_files(self, dataset_path=None, recursive=True, dataset_id=None):
        # type: (Optional[str], bool, Optional[str]) -> List[str]
        """
        returns a list of files in the current dataset
        If dataset_id is provided, return a list of files that remained unchanged since the specified dataset_version

        :param dataset_path: Only match files matching the dataset_path (including wildcards).
            Example: 'folder/sub/*.json'
        :param recursive: If True (default) matching dataset_path recursively
        :param dataset_id: Filter list based on the dataset id containing the latest version of the file.
            Default: None, do not filter files based on parent dataset.

        :return: List of files with relative path
            (files might not be available locally until get_local_copy() is called)
        """
        files = self._dataset_file_entries.keys() if not dataset_id else \
            [k for k, v in self._dataset_file_entries.items() if v.parent_dataset_id == dataset_id]

        if not dataset_path:
            return sorted(files)

        if dataset_path.startswith('/'):
            dataset_path = dataset_path[1:]

        if not recursive:
            return sorted([k for k in files if fnmatch(k + '/', dataset_path + '/')])

        wildcard = dataset_path.split('/')[-1]
        path = dataset_path[:-len(wildcard)] + '*'
        return sorted([k for k in files if fnmatch(k, path) and fnmatch(k, '*/' + wildcard)])

    def list_removed_files(self, dataset_id=None):
        # type: (str) -> List[str]
        """
        return a list of files removed when comparing to a specific dataset_version

        :param dataset_id: dataset id (str) to compare against, if None is given compare against the parents datasets
        :return: List of files with relative path
            (files might not be available locally until get_local_copy() is called)
        """
        datasets = self._dependency_graph[self._id] if not dataset_id or dataset_id == self._id else [dataset_id]
        unified_list = set()
        for ds_id in datasets:
            dataset = self.get(dataset_id=ds_id)
            unified_list |= set(dataset._dataset_file_entries.keys())

        removed_list = [f for f in unified_list if f not in self._dataset_file_entries]
        return sorted(removed_list)

    def list_modified_files(self, dataset_id=None):
        # type: (str) -> List[str]
        """
        return a list of files modified when comparing to a specific dataset_version

        :param dataset_id: dataset id (str) to compare against, if None is given compare against the parents datasets
        :return: List of files with relative path
            (files might not be available locally until get_local_copy() is called)
        """
        datasets = self._dependency_graph[self._id] if not dataset_id or dataset_id == self._id else [dataset_id]
        unified_list = dict()
        for ds_id in datasets:
            dataset = self.get(dataset_id=ds_id)
            unified_list.update(dict((k, v.hash) for k, v in dataset._dataset_file_entries.items()))

        modified_list = [k for k, v in self._dataset_file_entries.items()
                         if k in unified_list and v.hash != unified_list[k]]
        return sorted(modified_list)

    def list_added_files(self, dataset_id=None):
        # type: (str) -> List[str]
        """
        return a list of files added when comparing to a specific dataset_version

        :param dataset_id: dataset id (str) to compare against, if None is given compare against the parents datasets
        :return: List of files with relative path
            (files might not be available locally until get_local_copy() is called)
        """
        datasets = self._dependency_graph[self._id] if not dataset_id or dataset_id == self._id else [dataset_id]
        unified_list = set()
        for ds_id in datasets:
            dataset = self.get(dataset_id=ds_id)
            unified_list |= set(dataset._dataset_file_entries.keys())

        added_list = [f for f in self._dataset_file_entries.keys() if f not in unified_list]
        return sorted(added_list)

    def get_dependency_graph(self):
        """
        return the DAG of the dataset dependencies (all previous dataset version and their parents)

        Example:

        .. code-block:: py

            {
                'current_dataset_id': ['parent_1_id', 'parent_2_id'],
                'parent_2_id': ['parent_1_id'],
                'parent_1_id': [],
            }

        :return: dict representing the genealogy dag graph of the current dataset
        """
        return deepcopy(self._dependency_graph)

    def verify_dataset_hash(self, local_copy_path=None, skip_hash=False, verbose=False):
        # type: (Optional[str], bool, bool) -> List[str]
        """
        Verify the current copy of the dataset against the stored hash

        :param local_copy_path: Specify local path containing a copy of the dataset,
            If not provide use the cached folder
        :param skip_hash: If True, skip hash checks and verify file size only
        :param verbose: If True print errors while testing dataset files hash
        :return: List of files with unmatched hashes
        """
        local_path = local_copy_path or self.get_local_copy()

        def compare(file_entry):
            file_entry_copy = copy(file_entry)
            file_entry_copy.local_path = (Path(local_path) / file_entry.relative_path).as_posix()
            if skip_hash:
                file_entry_copy.size = Path(file_entry_copy.local_path).stat().st_size
                if file_entry_copy.size != file_entry.size:
                    if verbose:
                        print('Error: file size mismatch {} expected size {} current {}'.format(
                            file_entry.relative_path, file_entry.size, file_entry_copy.size))
                    return file_entry
            else:
                self._calc_file_hash(file_entry_copy)
                if file_entry_copy.hash != file_entry.hash:
                    if verbose:
                        print('Error: hash mismatch {} expected size/hash {}/{} recalculated {}/{}'.format(
                            file_entry.relative_path,
                            file_entry.size, file_entry.hash,
                            file_entry_copy.size, file_entry_copy.hash))
                    return file_entry

            return None

        pool = ThreadPool(cpu_count() * 2)
        matching_errors = pool.map(compare, self._dataset_file_entries.values())
        pool.close()
        return [f.relative_path for f in matching_errors if f is not None]

    def get_default_storage(self):
        # type: () -> Optional[str]
        """
        Return the default storage location of the dataset

        :return: URL for the default storage location
        """
        if not self._task:
            return None
        return self._task.output_uri or self._task.get_logger().get_default_upload_destination()

    @classmethod
    def create(
            cls,
            dataset_name=None,  # type: Optional[str]
            dataset_project=None,  # type: Optional[str]
            dataset_tags=None,  # type: Optional[Sequence[str]]
            parent_datasets=None,  # type: Optional[Sequence[Union[str, Dataset]]]
            use_current_task=False  # type: bool
    ):
        # type: (...) -> "Dataset"
        """
        Create a new dataset. Multiple dataset parents are supported.
        Merging of parent datasets is done based on the order,
        where each one can override overlapping files in the previous parent

        :param dataset_name: Naming the new dataset
        :param dataset_project: Project containing the dataset.
            If not specified, infer project name form parent datasets
        :param dataset_tags: Optional, list of tags (strings) to attach to the newly created Dataset
        :param parent_datasets: Expand a parent dataset by adding/removing files
        :param use_current_task: False (default), a new Dataset task is created.
            If True, the dataset is created on the current Task.
        :return: Newly created Dataset object
        """
        parent_datasets = [cls.get(dataset_id=p) if not isinstance(p, Dataset) else p for p in (parent_datasets or [])]
        if any(not p.is_final() for p in parent_datasets):
            raise ValueError("Cannot inherit from a parent that was not finalized/closed")

        # if dataset name + project are None, default to use current_task
        if dataset_project is None and dataset_name is None and not use_current_task:
            LoggerRoot.get_base_logger().info('New dataset project/name not provided, storing on Current Task')
            use_current_task = True

        # get project name
        if not dataset_project and not use_current_task:
            if not parent_datasets:
                raise ValueError("Missing dataset project name. Could not infer project name from parent dataset.")
            # get project name from parent dataset
            dataset_project = parent_datasets[-1]._task.get_project_name()

        # merge datasets according to order
        dataset_file_entries = {}
        dependency_graph = {}
        for p in parent_datasets:
            dataset_file_entries.update(deepcopy(p._dataset_file_entries))
            dependency_graph.update(deepcopy(p._dependency_graph))
        instance = cls(_private=cls.__private_magic,
                       dataset_project=dataset_project,
                       dataset_name=dataset_name,
                       dataset_tags=dataset_tags,
                       task=Task.current_task() if use_current_task else None)
        instance._using_current_task = use_current_task
        instance._task.get_logger().report_text('Dataset created', print_console=False)
        instance._dataset_file_entries = dataset_file_entries
        instance._dependency_graph = dependency_graph
        instance._dependency_graph[instance._id] = [p._id for p in parent_datasets]
        instance._serialize()
        instance._task.flush(wait_for_uploads=True)
        cls._set_project_system_tags(instance._task)
        return instance

    @classmethod
    def delete(cls, dataset_id=None, dataset_project=None, dataset_name=None, force=False):
        # type: (Optional[str], Optional[str], Optional[str], bool) -> ()
        """
        Delete a dataset, raise exception if dataset is used by other dataset versions.
        Use force=True to forcefully delete the dataset

        :param dataset_id: Dataset id to delete
        :param dataset_project: Project containing the dataset
        :param dataset_name: Naming the new dataset
        :param force: If True delete even if other datasets depend on the specified dataset version
        """
        mutually_exclusive(dataset_id=dataset_id, dataset_project=dataset_project)
        mutually_exclusive(dataset_id=dataset_id, dataset_name=dataset_name)
        if not dataset_id:
            tasks = Task.get_tasks(
                project_name=dataset_project,
                task_name=exact_match_regex(dataset_name) if dataset_name else None,
                task_filter=dict(
                    system_tags=[cls.__tag],
                    type=[str(Task.TaskTypes.data_processing)],
                    page_size=2, page=0,)
            )
            if not tasks:
                raise ValueError("Dataset project={} name={} could not be found".format(dataset_project, dataset_name))
            if len(tasks) > 1:
                raise ValueError("Too many datasets matching project={} name={}".format(dataset_project, dataset_name))
            dataset_id = tasks[0].id

        # check if someone is using the datasets
        if not force:
            # todo: use Task runtime_properties
            # noinspection PyProtectedMember
            dependencies = Task._query_tasks(
                system_tags=[cls.__tag],
                type=[str(Task.TaskTypes.data_processing)],
                only_fields=['created', 'id', 'name'],
                search_text='{}'.format(cls._get_dataset_id_hash(dataset_id))
            )
            # filter us out
            if dependencies:
                dependencies = [d for d in dependencies if d.id != dataset_id]
            if dependencies:
                raise ValueError("Dataset id={} is used by datasets: {}".format(
                    dataset_id, [d.id for d in dependencies]))

        client = APIClient()
        # notice the force here is a must, since the state is never draft
        # noinspection PyBroadException
        try:
            t = client.tasks.get_by_id(dataset_id)
        except Exception:
            t = None
        if not t:
            raise ValueError("Dataset id={} could not be found".format(dataset_id))
        if str(t.type) != str(Task.TaskTypes.data_processing) or cls.__tag not in t.system_tags:
            raise ValueError("Dataset id={} is not of type Dataset".format(dataset_id))

        task = Task.get_task(task_id=dataset_id)
        # first delete all the artifacts from the dataset
        for artifact in task.artifacts.values():
            h = StorageHelper.get(artifact.url)
            # noinspection PyBroadException
            try:
                h.delete(artifact.url)
            except Exception as ex:
                LoggerRoot.get_base_logger().warning('Failed deleting remote file \'{}\': {}'.format(
                    artifact.url, ex))

        # now delete the actual task
        client.tasks.delete(task=dataset_id, force=True)

    @classmethod
    def get(
            cls,
            dataset_id=None,  # type: Optional[str]
            dataset_project=None,  # type: Optional[str]
            dataset_name=None,  # type: Optional[str]
            dataset_tags=None,  # type: Optional[Sequence[str]]
            only_completed=False,  # type: bool
            only_published=False,  # type: bool
            auto_create=False,   # type: bool
            writable_copy=False  # type: bool
    ):
        # type: (...) -> "Dataset"
        """
        Get a specific Dataset. If only dataset_project is given, return the last Dataset in the Dataset project

        :param dataset_id: Requested Dataset ID
        :param dataset_project: Requested Dataset project name
        :param dataset_name: Requested Dataset name
        :param dataset_tags: Requested Dataset tags (list of tag strings)
        :param only_completed: Return only if the requested dataset is completed or published
        :param only_published: Return only if the requested dataset is published
        :param auto_create: Create new dataset if it does not exist yet
        :param writable_copy: Get a newly created mutable dataset with the current one as its parent,
            so new files can added to the instance.
        :return: Dataset object
        """
        mutually_exclusive(dataset_id=dataset_id, dataset_project=dataset_project, _require_at_least_one=False)
        mutually_exclusive(dataset_id=dataset_id, dataset_name=dataset_name, _require_at_least_one=False)
        if not any([dataset_id, dataset_project, dataset_name, dataset_tags]):
            raise ValueError("Dataset selection criteria not met. Didn't provide id/name/project/tags correctly.")

        if auto_create and not get_existing_project(
            session=Task._get_default_session(), project_name=dataset_project
        ):
            tasks = []
        else:
            tasks = Task.get_tasks(
                task_ids=[dataset_id] if dataset_id else None,
                project_name=dataset_project,
                task_name=exact_match_regex(dataset_name) if dataset_name else None,
                tags=dataset_tags,
                task_filter=dict(
                    system_tags=[cls.__tag, '-archived'], order_by=['-created'],
                    type=[str(Task.TaskTypes.data_processing)],
                    page_size=1, page=0,
                    status=['published'] if only_published else
                    ['published', 'completed', 'closed'] if only_completed else None)
            )

        if not tasks:
            if auto_create:
                instance = Dataset.create(dataset_name=dataset_name, dataset_project=dataset_project,
                                          dataset_tags=dataset_tags)
                return instance
            raise ValueError('Could not find Dataset {} {}'.format(
                'id' if dataset_id else 'project/name',
                dataset_id if dataset_id else (dataset_project, dataset_name)))
        task = tasks[0]
        if task.status == 'created':
            raise ValueError('Dataset id={} is in draft mode, delete and recreate it'.format(task.id))
        force_download = False if task.status in ('stopped', 'published', 'closed', 'completed') else True
        if cls.__state_entry_name in task.artifacts:
            local_state_file = StorageManager.get_local_copy(
                remote_url=task.artifacts[cls.__state_entry_name].url, cache_context=cls.__cache_context,
                extract_archive=False, name=task.id, force_download=force_download)
            if not local_state_file:
                raise ValueError('Could not load Dataset id={} state'.format(task.id))
        else:
            # we could not find the serialized state, start empty
            local_state_file = {}

        instance = cls._deserialize(local_state_file, task)
        # remove the artifact, just in case
        if force_download and local_state_file:
            os.unlink(local_state_file)

        # Now we have the requested dataset, but if we want a mutable copy instead, we create a new dataset with the
        # current one as its parent. So one can add files to it and finalize as a new version.
        if writable_copy:
            writeable_instance = Dataset.create(
                dataset_name=instance.name,
                dataset_project=instance.project,
                dataset_tags=instance.tags,
                parent_datasets=[instance.id],
            )
            return writeable_instance

        return instance

    def get_logger(self):
        # type: () -> Logger
        """
        Return a Logger object for the Dataset, allowing users to report statistics metrics
        and debug samples on the Dataset itself
        :return: Logger object
        """
        return self._task.get_logger()

    def get_num_chunks(self, include_parents=True):
        # type: (bool) -> int
        """
        Return the number of chunks stored on this dataset
        (it does not imply on the number of chunks parent versions store)

        :param include_parents: If True (default),
        return the total number of chunks from this version and all parent versions.
        If False, only return the number of chunks we stored on this specific version.

        :return: Number of chunks stored on the dataset.
        """
        if not include_parents:
            return len(self._get_data_artifact_names())

        return sum(self._get_dependency_chunk_lookup().values())

    @classmethod
    def squash(
            cls,
            dataset_name,  # type: str
            dataset_ids=None,  # type: Optional[Sequence[Union[str, Dataset]]]
            dataset_project_name_pairs=None,  # type: Optional[Sequence[(str, str)]]
            output_url=None,  # type: Optional[str]
    ):
        # type: (...) -> "Dataset"
        """
        Generate a new dataset from the squashed set of dataset versions.
        If a single version is given it will squash to the root (i.e. create single standalone version)
        If a set of versions are given it will squash the versions diff into a single version

        :param dataset_name: Target name for the newly generated squashed dataset
        :param dataset_ids: List of dataset Ids (or objects) to squash. Notice order does matter.
            The versions are merged from first to last.
        :param dataset_project_name_pairs: List of pairs (project_name, dataset_name) to squash.
            Notice order does matter. The versions are merged from first to last.
        :param output_url: Target storage for the compressed dataset (default: file server)
            Examples: `s3://bucket/data`, `gs://bucket/data` , `azure://bucket/data` , `/mnt/share/data`
        :return: Newly created dataset object.
        """
        mutually_exclusive(dataset_ids=dataset_ids, dataset_project_name_pairs=dataset_project_name_pairs)
        datasets = [cls.get(dataset_id=d) for d in dataset_ids] if dataset_ids else \
            [cls.get(dataset_project=pair[0], dataset_name=pair[1]) for pair in dataset_project_name_pairs]
        # single dataset to squash, squash it all.
        if len(datasets) == 1:
            temp_folder = datasets[0].get_local_copy()
            parents = set()
        else:
            parents = None
            temp_folder = Path(mkdtemp(prefix='squash-datasets.'))
            pool = ThreadPool()
            for ds in datasets:
                base_folder = Path(ds._extract_dataset_archive())
                files = [f.relative_path for f in ds.file_entries if f.parent_dataset_id == ds.id]
                pool.map(
                    lambda x:
                        (temp_folder / x).parent.mkdir(parents=True, exist_ok=True) or
                        shutil.copy((base_folder / x).as_posix(), (temp_folder / x).as_posix(), follow_symlinks=True),
                    files)
                parents = set(ds._get_parents()) if parents is None else (parents & set(ds._get_parents()))
            pool.close()

        squashed_ds = cls.create(
            dataset_project=datasets[0].project, dataset_name=dataset_name, parent_datasets=list(parents))
        squashed_ds._task.get_logger().report_text('Squashing dataset', print_console=False)
        squashed_ds.add_files(temp_folder)
        squashed_ds.upload(output_url=output_url)
        squashed_ds.finalize()
        return squashed_ds

    @classmethod
    def list_datasets(cls, dataset_project=None, partial_name=None, tags=None, ids=None, only_completed=True):
        # type: (Optional[str], Optional[str], Optional[Sequence[str]], Optional[Sequence[str]], bool) -> List[dict]
        """
        Query list of dataset in the system

        :param dataset_project: Specify dataset project name
        :param partial_name: Specify partial match to a dataset name
        :param tags: Specify user tags
        :param ids: List specific dataset based on IDs list
        :param only_completed: If False return dataset that are still in progress (uploading/edited etc.)
        :return: List of dictionaries with dataset information
            Example: [{'name': name, 'project': project name, 'id': dataset_id, 'created': date_created},]
        """
        # noinspection PyProtectedMember
        datasets = Task._query_tasks(
            task_ids=ids or None, project_name=dataset_project or None,
            task_name=partial_name,
            system_tags=[cls.__tag],
            type=[str(Task.TaskTypes.data_processing)],
            tags=tags or None,
            status=['stopped', 'published', 'completed', 'closed'] if only_completed else None,
            only_fields=['created', 'id', 'name', 'project', 'tags']
        )
        project_ids = {d.project for d in datasets}
        # noinspection PyProtectedMember
        project_id_lookup = {d: Task._get_project_name(d) for d in project_ids}
        return [
            {'name': d.name,
             'created': d.created,
             'project': project_id_lookup[d.project],
             'id': d.id,
             'tags': d.tags}
            for d in datasets
        ]

    def _add_files(self,
                   path,  # type: Union[str, Path, _Path]
                   wildcard=None,  # type: Optional[Union[str, Sequence[str]]]
                   local_base_folder=None,  # type: Optional[str]
                   dataset_path=None,  # type: Optional[str]
                   recursive=True,  # type: bool
                   verbose=False  # type: bool
                   ):
        # type: (...) -> int
        """
        Add a folder into the current dataset. calculate file hash,
        and compare against parent, mark files to be uploaded

        :param path: Add a folder/file to the dataset
        :param wildcard: add only specific set of files.
            Wildcard matching, can be a single string or a list of wildcards)
        :param local_base_folder: files will be located based on their relative path from local_base_folder
        :param dataset_path: where in the dataset the folder/files should be located
        :param recursive: If True match all wildcard files recursively
        :param verbose: If True print to console added files
        """
        if dataset_path and dataset_path.startswith('/'):
            dataset_path = dataset_path[1:]
        path = Path(path)
        local_base_folder = Path(local_base_folder or path)
        wildcard = wildcard or '*'
        # single file, no need for threading
        if path.is_file():
            if not local_base_folder.is_dir():
                local_base_folder = local_base_folder.parent
            file_entry = self._calc_file_hash(
                FileEntry(local_path=path.absolute().as_posix(),
                          relative_path=(Path(dataset_path or '.') / path.relative_to(local_base_folder)).as_posix(),
                          parent_dataset_id=self._id))
            file_entries = [file_entry]
        else:
            # if not a folder raise exception
            if not path.is_dir():
                raise ValueError("Could not find file/folder \'{}\'", path.as_posix())

            # prepare a list of files
            files = list(path.rglob(wildcard)) if recursive else list(path.glob(wildcard))
            file_entries = [
                FileEntry(
                    parent_dataset_id=self._id,
                    local_path=f.absolute().as_posix(),
                    relative_path=(Path(dataset_path or '.') / f.relative_to(local_base_folder)).as_posix())
                for f in files if f.is_file()]
            self._task.get_logger().report_text('Generating SHA2 hash for {} files'.format(len(file_entries)))
            pool = ThreadPool(cpu_count() * 2)
            try:
                import tqdm  # noqa
                for _ in tqdm.tqdm(pool.imap_unordered(self._calc_file_hash, file_entries), total=len(file_entries)):
                    pass
            except ImportError:
                pool.map(self._calc_file_hash, file_entries)
            pool.close()
            self._task.get_logger().report_text('Hash generation completed')

        # merge back into the dataset
        count = 0
        for f in file_entries:
            ds_cur_f = self._dataset_file_entries.get(f.relative_path)
            if not ds_cur_f:
                if verbose:
                    self._task.get_logger().report_text('Add {}'.format(f.relative_path))
                self._dataset_file_entries[f.relative_path] = f
                count += 1
            elif ds_cur_f.hash != f.hash:
                if verbose:
                    self._task.get_logger().report_text('Modified {}'.format(f.relative_path))
                self._dataset_file_entries[f.relative_path] = f
                count += 1
            elif f.parent_dataset_id == self._id and ds_cur_f.parent_dataset_id == self._id:
                # check if we have the file in an already uploaded chunk
                if ds_cur_f.local_path is None:
                    # skipping, already uploaded.
                    if verbose:
                        self._task.get_logger().report_text('Skipping {}'.format(f.relative_path))
                else:
                    # if we never uploaded it, mark for upload
                    if verbose:
                        self._task.get_logger().report_text('Re-Added {}'.format(f.relative_path))
                    self._dataset_file_entries[f.relative_path] = f
                    count += 1
            else:
                if verbose:
                    self._task.get_logger().report_text('Unchanged {}'.format(f.relative_path))

        return count

    def _update_dependency_graph(self):
        """
        Update the dependency graph based on the current self._dataset_file_entries state
        :return:
        """
        # collect all dataset versions
        used_dataset_versions = set(f.parent_dataset_id for f in self._dataset_file_entries.values())
        used_dataset_versions.add(self._id)
        current_parents = self._dependency_graph.get(self._id) or []
        # remove parent versions we no longer need from the main version list
        # per version, remove unnecessary parent versions, if we do not need them
        self._dependency_graph = {
            k: [p for p in parents or [] if p in used_dataset_versions]
            for k, parents in self._dependency_graph.items() if k in used_dataset_versions}
        # make sure we do not remove our parents, for geology sake
        self._dependency_graph[self._id] = current_parents

    def _serialize(self, update_dependency_chunk_lookup=False):
        # type: (bool) -> ()
        """
        store current state of the Dataset for later use

        :param update_dependency_chunk_lookup: If True, update the parent versions number of chunks

        :return: object to be used for later deserialization
        """
        self._update_dependency_graph()

        state = dict(
            dataset_file_entries=[f.as_dict() for f in self._dataset_file_entries.values()],
            dependency_graph=self._dependency_graph,
            id=self._id,
            dirty=self._dirty,
        )
        if update_dependency_chunk_lookup:
            state['dependency_chunk_lookup'] = self._build_dependency_chunk_lookup()

        modified_files = [f['size'] for f in state['dataset_file_entries'] if f.get('parent_dataset_id') == self._id]
        preview = \
            'Dataset state\n' \
            'Files added/modified: {0} - total size {1}\n' \
            'Current dependency graph: {2}\n'.format(
                len(modified_files), format_size(sum(modified_files)),
                json.dumps(self._dependency_graph, indent=2, sort_keys=True))
        # store as artifact of the Task.
        self._task.upload_artifact(
            name=self.__state_entry_name, artifact_object=state, preview=preview, wait_on_upload=True)

    def _download_dataset_archives(self):
        """
        Download the dataset archive, return a link to locally stored zip file
        :return: List of paths to locally stored zip files
        """
        pass  # TODO: implement

    def _extract_dataset_archive(
            self,
            force=False,
            selected_chunks=None,
            lock_target_folder=False,
            cleanup_target_folder=True,
            target_folder=None,
    ):
        # type: (bool, Optional[List[int]], bool, bool, Optional[Path]) -> str
        """
        Download the dataset archive, and extract the zip content to a cached folder.
        Notice no merging is done.

        :param force: If True extract dataset content even if target folder exists and is not empty
        :param selected_chunks: Optional, if provided only download the selected chunks (index) of the Dataset.
            Example: Assuming 8 chunks on this version
            selected_chunks=[0,1,2]
        :param lock_target_folder: If True, local the target folder so the next cleanup will not delete
            Notice you should unlock it manually, or wait fro the process to fnish for auto unlocking.
        :param cleanup_target_folder: If True remove target folder recursively
        :param target_folder: If provided use the specified target folder, default, auto generate from Dataset ID.

        :return: Path to a local storage extracted archive
        """
        assert selected_chunks is None or isinstance(selected_chunks, (list, tuple))

        if not self._task:
            self._task = Task.get_task(task_id=self._id)

        data_artifact_entries = self._get_data_artifact_names()

        if selected_chunks is not None and data_artifact_entries:
            data_artifact_entries = [
                d for d in data_artifact_entries
                if self._get_chunk_idx_from_artifact_name(d) in selected_chunks]

        # get cache manager
        local_folder = Path(target_folder) if target_folder else \
            self._create_ds_target_folder(lock_target_folder=lock_target_folder)

        # check if we have a dataset with empty change set
        if not data_artifact_entries:
            return local_folder.as_posix()

        # check if target folder is not empty
        if not force and next(local_folder.glob('*'), None):
            return local_folder.as_posix()

        # if we got here, we need to clear the target folder
        local_folder = local_folder.as_posix()
        if cleanup_target_folder:
            shutil.rmtree(local_folder, ignore_errors=True)
        # verify target folder exists
        Path(local_folder).mkdir(parents=True, exist_ok=True)

        def _download_part(data_artifact_name):
            # download the dataset zip
            local_zip = StorageManager.get_local_copy(
                remote_url=self._task.artifacts[data_artifact_name].url, cache_context=self.__cache_context,
                extract_archive=False, name=self._id)
            if not local_zip:
                raise ValueError("Could not download dataset id={} entry={}".format(self._id, data_artifact_name))
            # noinspection PyProtectedMember
            StorageManager._extract_to_cache(
                cached_file=local_zip, name=self._id,
                cache_context=self.__cache_context, target_folder=local_folder, force=True)

        # download al parts in parallel
        # if len(data_artifact_entries) > 1:
        #     pool = ThreadPool()
        #     pool.map(_download_part, data_artifact_entries)
        #     pool.close()
        # else:
        #     _download_part(data_artifact_entries[0])
        for d in data_artifact_entries:
            _download_part(d)

        return local_folder

    def _create_ds_target_folder(self, part=None, num_parts=None, lock_target_folder=True):
        # type: (Optional[int], Optional[int], bool) -> Path
        cache = CacheManager.get_cache_manager(cache_context=self.__cache_context)
        local_folder = Path(cache.get_cache_folder()) / self._get_cache_folder_name(part=part, num_parts=num_parts)
        if lock_target_folder:
            cache.lock_cache_folder(local_folder)
        local_folder.mkdir(parents=True, exist_ok=True)
        return local_folder

    def _get_data_artifact_names(self):
        # type: () -> List[str]
        data_artifact_entries = [
            a for a in self._task.artifacts
            if a and (a == self.__default_data_entry_name or str(a).startswith(self.__data_entry_name_prefix))]
        return data_artifact_entries

    def _get_next_data_artifact_name(self, last_artifact_name=None):
        # type: (Optional[str]) -> str
        if not last_artifact_name:
            data_artifact_entries = self._get_data_artifact_names()
            if len(data_artifact_entries) < 1:
                return self.__default_data_entry_name
        else:
            data_artifact_entries = [last_artifact_name]
        prefix = self.__data_entry_name_prefix
        prefix_len = len(prefix)
        numbers = sorted([int(a[prefix_len:]) for a in data_artifact_entries if a.startswith(prefix)])
        return '{}{:03d}'.format(prefix, numbers[-1]+1 if numbers else 1)

    def _merge_datasets(self, use_soft_links=None, raise_on_error=True, part=None, num_parts=None):
        # type: (bool, bool, Optional[int], Optional[int]) -> str
        """
        download and copy / soft-link, files from all the parent dataset versions
        :param use_soft_links: If True use soft links, default False on windows True on Posix systems
        :param raise_on_error: If True raise exception if dataset merging failed on any file
        :param part: Optional, if provided only download the selected part (index) of the Dataset.
            Notice, if `num_parts` is not provided, number of parts will be equal to the number of chunks.
            This argument is passed to parent versions, as well as the implicit `num_parts`,
            allowing users to get a partial copy of the entire dataset, for multi node/step processing.
        :param num_parts: Optional, If specified normalize the number of chunks stored to the
            requested number of parts. Notice that the actual chunks used per part are rounded down.
            Example: Assuming 8 chunks on this version, and `num_parts=5`, the chunk index used per parts would be:
            part=0 -> chunks[0,5], part=1 -> chunks[1,6], part=2 -> chunks[2,7], part=3 -> chunks[3, ]

        :return: the target folder
        """
        assert part is None or (isinstance(part, int) and part >= 0)
        assert num_parts is None or (isinstance(num_parts, int) and num_parts >= 1)

        if use_soft_links is None:
            use_soft_links = False if is_windows() else True

        if part is not None and not num_parts:
            num_parts = self.get_num_chunks()

        # just create the dataset target folder
        target_base_folder = self._create_ds_target_folder(
            part=part, num_parts=num_parts, lock_target_folder=True)

        # selected specific chunks if `part` was passed
        chunk_selection = None if part is None else self._build_chunk_selection(part=part, num_parts=num_parts)

        # check if target folder is not empty, see if it contains everything we need
        if target_base_folder and next(target_base_folder.iterdir(), None):
            if self._verify_dataset_folder(target_base_folder, part, chunk_selection):
                target_base_folder.touch()
                return target_base_folder.as_posix()
            else:
                LoggerRoot.get_base_logger().info('Dataset needs refreshing, fetching all parent datasets')
                # we should delete the entire cache folder
                shutil.rmtree(target_base_folder.as_posix())
                # make sure we recreate the dataset target folder
                target_base_folder.mkdir(parents=True, exist_ok=True)

        # get the dataset dependencies (if `part` was passed, only selected the ones in the selected part)
        dependencies_by_order = self._get_dependencies_by_order(include_unused=False, include_current=True) \
            if chunk_selection is None else list(chunk_selection.keys())

        # first get our dataset
        if self._id in dependencies_by_order:
            self._extract_dataset_archive(
                force=True,
                selected_chunks=chunk_selection.get(self._id) if chunk_selection else None,
                cleanup_target_folder=True,
                target_folder=target_base_folder,
            )
            dependencies_by_order.remove(self._id)

        # update target folder timestamp
        target_base_folder.touch()

        # if we have no dependencies, we can just return now
        if not dependencies_by_order:
            return target_base_folder.absolute().as_posix()

        # extract parent datasets
        self._extract_parent_datasets(
            target_base_folder=target_base_folder, dependencies_by_order=dependencies_by_order,
            chunk_selection=chunk_selection, use_soft_links=use_soft_links,
            raise_on_error=False, force=False)

        # verify entire dataset (if failed, force downloading parent datasets)
        if not self._verify_dataset_folder(target_base_folder, part, chunk_selection):
            LoggerRoot.get_base_logger().info('Dataset parents need refreshing, re-fetching all parent datasets')
            # we should delete the entire cache folder
            self._extract_parent_datasets(
                target_base_folder=target_base_folder, dependencies_by_order=dependencies_by_order,
                chunk_selection=chunk_selection, use_soft_links=use_soft_links,
                raise_on_error=raise_on_error, force=True)

        return target_base_folder.absolute().as_posix()

    def _get_dependencies_by_order(self, include_unused=False, include_current=True):
        # type: (bool, bool) -> List[str]
        """
        Return the dataset dependencies by order of application (from the last to the current)
        :param include_unused: If True include unused datasets in the dependencies
        :param include_current: If True include the current dataset ID as the last ID in the list
        :return: list of str representing the datasets id
        """
        roots = [self._id]
        dependencies = []
        # noinspection DuplicatedCode
        while roots:
            r = roots.pop(0)
            if r not in dependencies:
                dependencies.append(r)
            # add the parents of the current node, only if the parents are in the general graph node list
            if include_unused and r not in self._dependency_graph:
                roots.extend(list(reversed(
                    [p for p in (self.get(dataset_id=r)._get_parents() or []) if p not in roots])))
            else:
                roots.extend(list(reversed(
                    [p for p in (self._dependency_graph.get(r) or [])
                     if p not in roots and (include_unused or (p in self._dependency_graph))])))

        # make sure we cover leftovers
        leftovers = set(self._dependency_graph.keys()) - set(dependencies)
        if leftovers:
            roots = list(leftovers)
            # noinspection DuplicatedCode
            while roots:
                r = roots.pop(0)
                if r not in dependencies:
                    dependencies.append(r)
                # add the parents of the current node, only if the parents are in the general graph node list
                if include_unused and r not in self._dependency_graph:
                    roots.extend(list(reversed(
                        [p for p in (self.get(dataset_id=r)._get_parents() or []) if p not in roots])))
                else:
                    roots.extend(list(reversed(
                        [p for p in (self._dependency_graph.get(r) or [])
                         if p not in roots and (include_unused or (p in self._dependency_graph))])))

        # skip our id
        dependencies = list(reversed(dependencies[1:]))
        return (dependencies + [self._id]) if include_current else dependencies

    def _get_parents(self):
        # type: () -> Sequence[str]
        """
        Return a list of direct parent datasets (str)
        :return: list of dataset ids
        """
        return self._dependency_graph[self.id]

    @classmethod
    def _deserialize(cls, stored_state, task):
        # type: (Union[dict, str, Path, _Path], Task) -> "Dataset"
        """
        reload a dataset state from the stored_state object
        :param task: Task object associated with the dataset
        :return: A Dataset object
        """
        assert isinstance(stored_state, (dict, str, Path, _Path))

        if isinstance(stored_state, (str, Path, _Path)):
            stored_state_file = Path(stored_state).as_posix()
            with open(stored_state_file, 'rt') as f:
                stored_state = json.load(f)

        instance = cls(_private=cls.__private_magic, task=task)
        # assert instance._id == stored_state['id']  # They should match
        instance._dependency_graph = stored_state.get('dependency_graph', {})
        instance._dirty = stored_state.get('dirty', False)
        instance._dataset_file_entries = {
            s['relative_path']: FileEntry(**s) for s in stored_state.get('dataset_file_entries', [])}
        if stored_state.get('dependency_chunk_lookup') is not None:
            instance._dependency_chunk_lookup = stored_state.get('dependency_chunk_lookup')

        # update the last used artifact (remove the one we never serialized, they rae considered broken)
        if task.status in ('in_progress', 'created', 'stopped'):
            artifact_names = set([
                a.artifact_name for a in instance._dataset_file_entries.values()
                if a.artifact_name and a.parent_dataset_id == instance._id])
            missing_artifact_name = set(instance._get_data_artifact_names()) - artifact_names
            if missing_artifact_name:
                instance._task._delete_artifacts(list(missing_artifact_name))
                # if we removed any data artifact, update the next data artifact name
                instance._data_artifact_name = instance._get_next_data_artifact_name()

        return instance

    @staticmethod
    def _calc_file_hash(file_entry):
        # calculate hash
        file_entry.hash, _ = sha256sum(file_entry.local_path)
        file_entry.size = Path(file_entry.local_path).stat().st_size
        return file_entry

    @classmethod
    def _get_dataset_id_hash(cls, dataset_id):
        # type: (str) -> str
        """
        Return hash used to search for the dataset id in text fields.
        This is not a strong hash and used for defining dependencies.
        :param dataset_id:
        :return:
        """
        return 'dsh{}'.format(md5text(dataset_id))

    def _build_dependency_chunk_lookup(self):
        # type: () -> Dict[str, int]
        """
        Build the dependency dataset id to number-of-chunks, lookup table
        :return: lookup dictionary from dataset-id to number of chunks
        """
        # with ThreadPool() as pool:
        #     chunks_lookup = pool.map(
        #         lambda d: (d, Dataset.get(dataset_id=d).get_num_chunks()),
        #         self._dependency_graph.keys())
        #     return dict(chunks_lookup)
        chunks_lookup = map(
            lambda d: (d, (self if d == self.id else Dataset.get(dataset_id=d)).get_num_chunks(include_parents=False)),
            self._dependency_graph.keys())
        return dict(chunks_lookup)

    def _get_cache_folder_name(self, part=None, num_parts=None):
        # type: (Optional[int], Optional[int]) -> str
        if part is None:
            return '{}{}'.format(self.__cache_folder_prefix, self._id)
        return '{}{}_{}_{}'.format(self.__cache_folder_prefix, self._id, part, num_parts)

    def _add_script_call(self, func_name, **kwargs):
        # type: (str, **Any) -> ()

        # if we never created the Task, we should not add the script calls
        if not self._created_task:
            return

        args = ', '.join('\n    {}={}'.format(k, '\''+str(v)+'\'' if isinstance(v, (str, Path, _Path)) else v)
                         for k, v in kwargs.items())
        if args:
            args += '\n'
        line = 'ds.{}({})\n'.format(func_name, args)
        self._task.data.script.diff += line
        # noinspection PyProtectedMember
        self._task._edit(script=self._task.data.script)

    def _report_dataset_genealogy(self):
        sankey_node = dict(
            label=[],
            color=[],
            customdata=[],
            hovertemplate='%{customdata}<extra></extra>',
            hoverlabel={"align": "left"},
        )
        sankey_link = dict(
            source=[],
            target=[],
            value=[],
            hovertemplate='<extra></extra>',
        )
        # get DAG nodes
        nodes = self._get_dependencies_by_order(include_unused=True, include_current=True)
        # dataset name lookup
        # noinspection PyProtectedMember
        node_names = {t.id: t.name for t in Task._query_tasks(task_ids=nodes, only_fields=['id', 'name'])}
        node_details = {}
        # Generate table and details
        table_values = [["Dataset id", "name", "removed", "modified", "added", "size"]]
        for node in nodes:
            count = 0
            size = 0
            for f in self._dataset_file_entries.values():
                if f.parent_dataset_id == node:
                    count += 1
                    size += f.size
            removed = len(self.list_removed_files(node))
            modified = len(self.list_modified_files(node))
            table_values += [[node, node_names.get(node, ''),
                              removed, modified, max(0, count-modified), format_size(size)]]
            node_details[node] = [removed, modified, max(0, count-modified), format_size(size)]

        # create DAG
        visited = []
        # add nodes
        for idx, node in enumerate(nodes):
            visited.append(node)
            sankey_node['color'].append("mediumpurple" if node == self.id else "lightblue")
            sankey_node['label'].append('{}'.format(node))
            sankey_node['customdata'].append(
                "name {}<br />removed {}<br />modified {}<br />added {}<br />size {}".format(
                    node_names.get(node, ''), *node_details[node]))

        # add edges
        for idx, node in enumerate(nodes):
            if node in self._dependency_graph:
                parents = [visited.index(p) for p in self._dependency_graph[node] or [] if p in visited]
            else:
                parents = [visited.index(p) for p in self.get(dataset_id=node)._get_parents() or [] if p in visited]

            for p in parents:
                sankey_link['source'].append(p)
                sankey_link['target'].append(idx)
                sankey_link['value'].append(max(1, node_details[visited[p]][-2]))

        if len(nodes) > 1:
            # create the sankey graph
            dag_flow = dict(
                link=sankey_link,
                node=sankey_node,
                textfont=dict(color='rgba(0,0,0,255)', size=10),
                type='sankey',
                orientation='h'
            )
            fig = dict(data=[dag_flow], layout={'xaxis': {'visible': False}, 'yaxis': {'visible': False}})
        elif len(nodes) == 1:
            # hack, show single node sankey
            singles_flow = dict(
                x=list(range(len(nodes))), y=[1] * len(nodes),
                text=sankey_node['label'],
                customdata=sankey_node['customdata'],
                mode='markers',
                hovertemplate='%{customdata}<extra></extra>',
                marker=dict(
                    color=[v for i, v in enumerate(sankey_node['color']) if i in nodes],
                    size=[40] * len(nodes),
                ),
                showlegend=False,
                type='scatter',
            )
            # only single nodes
            fig = dict(data=[singles_flow], layout={
                'hovermode': 'closest', 'xaxis': {'visible': False}, 'yaxis': {'visible': False}})
        else:
            fig = None

        # report genealogy
        if fig:
            self._task.get_logger().report_plotly(
                title='Dataset Genealogy', series='', iteration=0, figure=fig)

        # report detailed table
        self._task.get_logger().report_table(
            title='Dataset Summary', series='Details', iteration=0, table_plot=table_values,
            extra_layout={"title": "Files by parent dataset id"})

        # report the detailed content of the dataset as configuration,
        # this allows for easy version comparison in the UI
        dataset_details = None
        dataset_details_header = None
        dataset_details_header_template = 'File Name ({} files) - File Size (total {}) - Hash (SHA2)\n'
        if len(self._dataset_file_entries) < self.__preview_max_file_entries:
            file_entries = sorted(self._dataset_file_entries.values(), key=lambda x: x.relative_path)
            dataset_details_header = dataset_details_header_template.format(
                    len(file_entries), format_size(sum(f.size for f in file_entries))
                )
            dataset_details = dataset_details_header + \
                '\n'.join('{} - {} - {}'.format(f.relative_path, f.size, f.hash) for f in file_entries)

        # too large to store
        if not dataset_details or len(dataset_details) > self.__preview_max_size:
            if not dataset_details_header:
                dataset_details_header = dataset_details_header_template.format(
                    len(self._dataset_file_entries),
                    format_size(sum(f.size for f in self._dataset_file_entries.values()))
                )
            dataset_details = dataset_details_header + 'Dataset content is too large to preview'

        # noinspection PyProtectedMember
        self._task._set_configuration(
            name='Dataset Content', description='Dataset content preview',
            config_type='read-only',
            config_text=dataset_details
        )

    @classmethod
    def _set_project_system_tags(cls, task):
        from ..backend_api.services import projects
        res = task.send(projects.GetByIdRequest(project=task.project), raise_on_errors=False)
        if not res or not res.response or not res.response.project:
            return
        system_tags = res.response.project.system_tags or []
        if cls.__tag not in system_tags:
            system_tags += [cls.__tag]
            task.send(projects.UpdateRequest(project=task.project, system_tags=system_tags), raise_on_errors=False)

    def is_dirty(self):
        # type: () -> bool
        """
        Return True if the dataset has pending uploads (i.e. we cannot finalize it)

        :return: Return True means dataset has pending uploads, call 'upload' to start an upload process.
        """
        return self._dirty

    def _extract_parent_datasets(
            self,
            target_base_folder,
            dependencies_by_order,
            chunk_selection,
            use_soft_links,
            raise_on_error,
            force
    ):
        # type: (Path, List[str], Optional[dict], bool, bool, bool) -> ()
        # create thread pool, for creating soft-links / copying
        # todo: parallelize by parent datasets
        pool = ThreadPool(cpu_count() * 2)
        for dataset_version_id in dependencies_by_order:
            # make sure we skip over empty dependencies
            if dataset_version_id not in self._dependency_graph:
                continue
            selected_chunks = chunk_selection.get(dataset_version_id) if chunk_selection else None

            ds = Dataset.get(dataset_id=dataset_version_id)
            ds_base_folder = Path(ds._extract_dataset_archive(
                selected_chunks=selected_chunks,
                force=force,
                lock_target_folder=True,
                cleanup_target_folder=False,
            ))
            ds_base_folder.touch()

            def copy_file(file_entry):
                if file_entry.parent_dataset_id != dataset_version_id or \
                        (selected_chunks is not None and
                         self._get_chunk_idx_from_artifact_name(file_entry.artifact_name) not in selected_chunks):
                    return
                source = (ds_base_folder / file_entry.relative_path).as_posix()
                target = (target_base_folder / file_entry.relative_path).as_posix()
                try:
                    # make sure we have can overwrite the target file
                    # noinspection PyBroadException
                    try:
                        os.unlink(target)
                    except Exception:
                        Path(target).parent.mkdir(parents=True, exist_ok=True)

                    # copy / link
                    if use_soft_links:
                        if not os.path.isfile(source):
                            raise ValueError("Extracted file missing {}".format(source))
                        os.symlink(source, target)
                    else:
                        shutil.copy2(source, target, follow_symlinks=True)
                except Exception as ex:
                    LoggerRoot.get_base_logger().warning('{}\nFailed {} file {} to {}'.format(
                        ex, 'linking' if use_soft_links else 'copying', source, target))
                    return ex

                return None

            errors = pool.map(copy_file, self._dataset_file_entries.values())

            CacheManager.get_cache_manager(cache_context=self.__cache_context).unlock_cache_folder(
                ds_base_folder.as_posix())

            if raise_on_error and any(errors):
                raise ValueError("Dataset merging failed: {}".format([e for e in errors if e is not None]))
        pool.close()

    def _verify_dataset_folder(self, target_base_folder, part, chunk_selection):
        # type: (Path, Optional[int], Optional[dict]) -> bool
        target_base_folder = Path(target_base_folder)
        # check dataset file size, if we have a full match no need for parent dataset download / merge
        verified = True
        # noinspection PyBroadException
        try:
            for f in self._dataset_file_entries.values():
                # check if we need it for the current part
                if part is not None:
                    f_parts = chunk_selection.get(f.parent_dataset_id, [])
                    # this is not in our current part, no need to check it.
                    if self._get_chunk_idx_from_artifact_name(f.artifact_name) not in f_parts:
                        continue

                # check if the local size and the stored size match (faster than comparing hash)
                if (target_base_folder / f.relative_path).stat().st_size != f.size:
                    verified = False
                    break

        except Exception:
            verified = False

        return verified

    def _get_dependency_chunk_lookup(self):
        # type: () -> Dict[str, int]
        """
        Return The parent dataset ID to number of chunks lookup table
        :return: Dict key is dataset ID, value is total number of chunks for the specific dataset version.
        """
        if self._dependency_chunk_lookup is None:
            self._dependency_chunk_lookup = self._build_dependency_chunk_lookup()
        return self._dependency_chunk_lookup

    def _build_chunk_selection(self, part, num_parts):
        # type: (int, int) -> Dict[str, int]
        """
        Build the selected chunks from each parent version, based on the current selection.
        Notice that for a specific part, one can only get the chunks from parent versions (not including this one)
        :param part: Current part index (between 0 and num_parts-1)
        :param num_parts: Total number of parts to divide the dataset into
        :return: Dict of Dataset ID and their respected chunks used for this part number
        """
        # get the chunk dependencies
        dependency_chunk_lookup = self._get_dependency_chunk_lookup()

        # first collect the total number of chunks
        total_chunks = sum(dependency_chunk_lookup.values())

        avg_chunk_per_part = total_chunks // num_parts
        leftover_chunks = total_chunks % num_parts

        dependencies = self._get_dependencies_by_order(include_unused=False, include_current=True)
        # create the part look up
        ds_id_chunk_list = [(d, i) for d in dependencies for i in range(dependency_chunk_lookup.get(d, 1))]

        # select the chunks for this part
        if part < leftover_chunks:
            indexes = ds_id_chunk_list[part*(avg_chunk_per_part+1):(part+1)*(avg_chunk_per_part+1)]
        else:
            ds_id_chunk_list = ds_id_chunk_list[leftover_chunks*(avg_chunk_per_part+1):]
            indexes = ds_id_chunk_list[(part-leftover_chunks)*avg_chunk_per_part:
                                       (part-leftover_chunks+1)*avg_chunk_per_part]

        # convert to lookup
        chunk_selection = {}
        for d, i in indexes:
            chunk_selection[d] = chunk_selection.get(d, []) + [i]

        return chunk_selection

    @classmethod
    def _get_chunk_idx_from_artifact_name(cls, artifact_name):
        # type: (str) -> int
        if not artifact_name:
            return -1
        artifact_name = str(artifact_name)

        if artifact_name == cls.__default_data_entry_name:
            return 0
        if artifact_name.startswith(cls.__data_entry_name_prefix):
            return int(artifact_name[len(cls.__data_entry_name_prefix):])
        return -1


# clearml/debugging/__init__.py

""" Debugging module """
from .timer import Timer
from .log import get_logger, get_null_logger, TqdmLog, add_options as add_log_options, \
    apply_logging_args as parse_log_args, add_rotating_file_handler, add_time_rotating_file_handler

__all__ = ["Timer", "get_logger", "get_null_logger", "TqdmLog", "add_log_options", "parse_log_args",
           "add_rotating_file_handler", "add_time_rotating_file_handler"]


# clearml/debugging/log.py

""" Logging convenience functions and wrappers """
import inspect
import logging
import logging.handlers
import os
import sys
from platform import system

from pathlib2 import Path
from six import BytesIO

default_level = logging.INFO


class PickledLogger(logging.getLoggerClass()):

    def __init__(self, *args, **kwargs):
        super(PickledLogger, self).__init__(*args, **kwargs)
        self._init_kwargs = None

    @staticmethod
    def wrapper(a_instance, func, **kwargs):
        # if python 3.7 and above Loggers are pickle-able
        if sys.version_info.major >= 3 and sys.version_info.minor >= 7:
            return a_instance

        safe_logger = PickledLogger(name=kwargs.get('name'))
        safe_logger.__dict__ = a_instance.__dict__
        if 'stream' in kwargs and kwargs['stream']:
            kwargs['stream'] = 'stdout' if kwargs['stream'] == sys.stdout else (
                'stderr' if kwargs['stream'] == sys.stderr else kwargs['stream'])
        else:
            kwargs['stream'] = None
        kwargs['_func'] = func
        safe_logger._init_kwargs = kwargs
        return safe_logger

    def __getstate__(self):
        return self._init_kwargs or {}

    def __setstate__(self, state):
        state['stream'] = sys.stdout if state['stream'] == 'stdout' else (
            sys.stderr if state['stream'] == 'stderr' else state['stream'])
        _func = state.pop('_func') or self.__class__
        self.__dict__ = _func(**state).__dict__


class _LevelRangeFilter(logging.Filter):

    def __init__(self, min_level, max_level, name=''):
        super(_LevelRangeFilter, self).__init__(name)
        self.min_level = min_level
        self.max_level = max_level

    def filter(self, record):
        return self.min_level <= record.levelno <= self.max_level


class LoggerRoot(object):
    __base_logger = None

    @classmethod
    def _make_stream_handler(cls, level=None, stream=sys.stdout, colored=False):
        ch = logging.StreamHandler(stream=stream)
        ch.setLevel(level)
        formatter = None

        # if colored, try to import colorama & coloredlogs (by default, not in the requirements)
        if colored:
            try:
                import colorama
                from coloredlogs import ColoredFormatter
                colorama.init()
                formatter = ColoredFormatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
            except ImportError:
                colored = False

        # if we don't need or failed getting colored formatter
        if not colored or not formatter:
            formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

        ch.setFormatter(formatter)
        return ch

    @classmethod
    def get_base_logger(cls, level=None, stream=sys.stdout, colored=False):
        if LoggerRoot.__base_logger:
            return LoggerRoot.__base_logger
        # avoid nested imports
        from ..config import get_log_redirect_level
        LoggerRoot.__base_logger = PickledLogger.wrapper(
            logging.getLogger('clearml'),
            func=cls.get_base_logger,
            level=level, stream=stream, colored=colored)
        level = level if level is not None else default_level
        LoggerRoot.__base_logger.setLevel(level)

        redirect_level = get_log_redirect_level()

        # Do not redirect to stderr if the target stream is already stderr
        if redirect_level is not None and stream not in (None, sys.stderr):
            # Adjust redirect level in case requested level is higher (e.g. logger is requested for CRITICAL
            # and redirect is set for ERROR, in which case we redirect from CRITICAL)
            redirect_level = max(level, redirect_level)
            LoggerRoot.__base_logger.addHandler(
                cls._make_stream_handler(redirect_level, sys.stderr, colored)
            )

            if level < redirect_level:
                # Not all levels were redirected, remaining should be sent to requested stream
                handler = cls._make_stream_handler(level, stream, colored)
                handler.addFilter(_LevelRangeFilter(min_level=level, max_level=redirect_level - 1))
                LoggerRoot.__base_logger.addHandler(handler)
        else:
            LoggerRoot.__base_logger.addHandler(
                cls._make_stream_handler(level, stream, colored)
            )

        LoggerRoot.__base_logger.propagate = False
        return LoggerRoot.__base_logger

    @classmethod
    def flush(cls):
        if LoggerRoot.__base_logger:
            for h in LoggerRoot.__base_logger.handlers:
                h.flush()


def add_options(parser):
    """ Add logging options to an argparse.ArgumentParser object """
    level = logging.getLevelName(default_level)
    parser.add_argument(
        '--log-level', '-l', default=level, help='Log level (default is %s)' % level)


def apply_logging_args(args):
    """ Apply logging args from an argparse.ArgumentParser parsed args """
    global default_level
    default_level = logging.getLevelName(args.log_level.upper())


def get_logger(path=None, level=None, stream=None, colored=False):
    """ Get a python logging object named using the provided filename and preconfigured with a color-formatted
        stream handler
    """
    # noinspection PyBroadException
    try:
        path = path or os.path.abspath((inspect.stack()[1])[1])
    except BaseException:
        # if for some reason we could not find the calling file, use our own
        path = os.path.abspath(__file__)
    root_log = LoggerRoot.get_base_logger(level=default_level, stream=sys.stdout, colored=colored)
    log = root_log.getChild(Path(path).stem)
    if level is not None:
        log.setLevel(level)
    if stream:
        ch = logging.StreamHandler(stream=stream)
        if level is not None:
            ch.setLevel(level)
    log.propagate = True
    return PickledLogger.wrapper(
        log, func=get_logger, path=path, level=level, stream=stream, colored=colored)


def _add_file_handler(logger, log_dir, fh, formatter=None):
    """ Adds a file handler to a logger """
    Path(log_dir).mkdir(parents=True, exist_ok=True)
    if not formatter:
        log_format = '%(asctime)s %(name)s x_x[%(levelname)s] %(message)s'
        formatter = logging.Formatter(log_format)
    fh.setFormatter(formatter)
    logger.addHandler(fh)


def add_rotating_file_handler(logger, log_dir, log_file_prefix, max_bytes=10 * 1024 * 1024, backup_count=20,
                              formatter=None):
    """ Create and add a rotating file handler to a logger """
    fh = logging.handlers.RotatingFileHandler(
        str(Path(log_dir) / ('%s.log' % log_file_prefix)), maxBytes=max_bytes, backupCount=backup_count)
    _add_file_handler(logger, log_dir, fh, formatter)


def add_time_rotating_file_handler(logger, log_dir, log_file_prefix, when='midnight', formatter=None):
    """
        Create and add a time rotating file handler to a logger.
        Possible values for when are 'midnight', weekdays ('w0'-'W6', when 0 is Monday), and 's', 'm', 'h' amd 'd' for
            seconds, minutes, hours and days respectively (case-insensitive)
    """
    fh = logging.handlers.TimedRotatingFileHandler(
        str(Path(log_dir) / ('%s.log' % log_file_prefix)), when=when)
    _add_file_handler(logger, log_dir, fh, formatter)


def get_null_logger(name=None):
    """ Get a logger with a null handler """
    log = logging.getLogger(name if name else 'null')
    if not log.handlers:
        # avoid nested imports
        from ..config import config

        log.addHandler(logging.NullHandler())
        log.propagate = config.get("log.null_log_propagate", False)
    return PickledLogger.wrapper(log, func=get_null_logger, name=name)


class TqdmLog(object):
    """ Tqdm (progressbar) wrapped logging class """

    class _TqdmIO(BytesIO):
        """ IO wrapper class for Tqdm """

        def __init__(self, level=20, logger=None, *args, **kwargs):
            self._log = logger or get_null_logger()
            self._level = level
            BytesIO.__init__(self, *args, **kwargs)

        def write(self, buf):
            self._buf = buf.strip('\r\n\t ')

        def flush(self):
            self._log.log(self._level, self._buf)

    def __init__(self, total, desc='', log_level=20, ascii=False, logger=None, smoothing=0, mininterval=5, initial=0):
        from tqdm import tqdm

        self._io = self._TqdmIO(level=log_level, logger=logger)
        self._tqdm = tqdm(total=total, desc=desc, file=self._io, ascii=ascii if not system() == 'Windows' else True,
                          smoothing=smoothing,
                          mininterval=mininterval, initial=initial)

    def update(self, n=None):
        if n is not None:
            self._tqdm.update(n=n)
        else:
            self._tqdm.update()

    def close(self):
        self._tqdm.close()


# clearml/debugging/timer.py

""" Timing support """
import sys
import time

import six


class Timer(object):
    """A class implementing a simple timer, with a reset option """

    def __init__(self):
        self._start_time = 0.
        self._diff = 0.
        self._total_time = 0.
        self._average_time = 0.
        self._calls = 0
        self.tic()

    def reset(self):
        self._start_time = 0.
        self._diff = 0.
        self.reset_average()

    def reset_average(self):
        """ Reset average counters (does not change current timer) """
        self._total_time = 0
        self._average_time = 0
        self._calls = 0

    def tic(self):
        try:
            # using time.time instead of time.clock because time time.clock
            # does not normalize for multi threading
            self._start_time = time.time()
        except Exception:
            pass

    def toc(self, average=True):
        self._diff = time.time() - self._start_time
        self._total_time += self._diff
        self._calls += 1
        self._average_time = self._total_time / self._calls
        if average:
            return self._average_time
        else:
            return self._diff

    @property
    def average_time(self):
        return self._average_time

    @property
    def total_time(self):
        return self._total_time

    def toc_with_reset(self, average=True, reset_if_calls=1000):
        """ Enable toc with reset (slightly inaccurate if reset event occurs) """
        if self._calls > reset_if_calls:
            last_diff = time.time() - self._start_time
            self._start_time = time.time()
            self._total_time = last_diff
            self._average_time = 0
            self._calls = 0

        return self.toc(average=average)


class TimersMixin(object):
    def __init__(self):
        self._timers = {}

    def add_timers(self, *names):
        for name in names:
            self.add_timer(name)

    def add_timer(self, name, timer=None):
        if name in self._timers:
            raise ValueError('timer %s already exists' % name)
        timer = timer or Timer()
        self._timers[name] = timer
        return timer

    def get_timer(self, name, default=None):
        return self._timers.get(name, default)

    def get_timers(self):
        return self._timers

    def _call_timer(self, name, callable, silent_fail=False):
        try:
            return callable(self._timers[name])
        except KeyError:
            if not silent_fail:
                six.reraise(*sys.exc_info())

    def reset_timers(self, *names):
        for name in names:
            self._call_timer(name, lambda t: t.reset())

    def reset_average_timers(self, *names):
        for name in names:
            self._call_timer(name, lambda t: t.reset_average())

    def tic_timers(self, *names):
        for name in names:
            self._call_timer(name, lambda t: t.tic())

    def toc_timers(self, *names):
        return [self._call_timer(name, lambda t: t.toc()) for name in names]

    def toc_with_reset_timer(self, name, average=True, reset_if_calls=1000):
        return self._call_timer(name, lambda t: t.toc_with_reset(average, reset_if_calls))


# clearml/debugging/trace.py

import os
import sys
import threading
import inspect
import time
import zipfile

__stream_write = None
__stream_flush = None
__patched_trace = False
__trace_level = 1
__trace_start = 0
__thread_id = None
__thread_so = None


def _thread_linux_id():
    # System dependent, see e.g. /usr/include/x86_64-linux-gnu/asm/unistd_64.h (system call 186)
    return __thread_so.syscall(186)


def _thread_py_id():
    # return threading.get_ident()
    return zipfile.crc32(int(threading.get_ident()).to_bytes(8, 'little'))


def _log_stderr(name, fnc, args, kwargs, is_return):
    global __stream_write, __stream_flush, __trace_level, __trace_start, __thread_id
    # noinspection PyBroadException
    try:
        if is_return and __trace_level not in (-1, -2):
            return
        if __trace_level not in (1, 2, -1, -2):
            return
        fnc_address = str(fnc).split(' at ')
        fnc_address = '{}'.format(fnc_address[-1].replace('>', '')) if len(fnc_address) > 1 else ''
        if __trace_level == 1 or __trace_level == -1:
            t = '{:14} {}'.format(fnc_address, name)
        elif __trace_level == 2 or __trace_level == -2:
            a_args = str(args)[1:-1] if args else ''
            a_kwargs = ' {}'.format(kwargs) if kwargs else ''
            t = '{:14} {} ({}{})'.format(fnc_address, name, a_args, a_kwargs)
        # get a nicer thread id
        h = int(__thread_id())
        ts = time.time() - __trace_start
        __stream_write('{}{:<9.3f}:{:5}:{:8x}: [{}] {}\n'.format(
            '-' if is_return else '', ts, os.getpid(),
            h, threading.current_thread().name, t))
        if __stream_flush:
            __stream_flush()
    except Exception:
        pass


def _traced_call_method(name, fnc):
    def _traced_call_int(self, *args, **kwargs):
        _log_stderr(name, fnc, args, kwargs, False)
        r = None
        try:
            ret = fnc(self, *args, **kwargs)
        except Exception as ex:
            r = ex
        _log_stderr(name, fnc, args, kwargs, True)
        if r:
            raise r
        return ret
    return _traced_call_int


def _traced_call_cls(name, fnc):
    class WrapperClass(object):
        @classmethod
        def _traced_call_int(cls, *args, **kwargs):
            _log_stderr(name, fnc, args, kwargs, False)
            r = None
            try:
                ret = fnc(*args, **kwargs)
            except Exception as ex:
                r = ex
            _log_stderr(name, fnc, args, kwargs, True)
            if r:
                raise r
            return ret

    return WrapperClass.__dict__['_traced_call_int']


def _traced_call_static(name, fnc):
    class WrapperStatic(object):
        @staticmethod
        def _traced_call_int(*args, **kwargs):
            _log_stderr(name, fnc, args, kwargs, False)
            r = None
            try:
                ret = fnc(*args, **kwargs)
            except Exception as ex:
                r = ex
            _log_stderr(name, fnc, args, kwargs, True)
            if r:
                raise r
            return ret
    return WrapperStatic.__dict__['_traced_call_int']


def _traced_call_func(name, fnc):
    def _traced_call_int(*args, **kwargs):
        _log_stderr(name, fnc, args, kwargs, False)
        r = None
        try:
            ret = fnc(*args, **kwargs)
        except Exception as ex:
            r = ex
        _log_stderr(name, fnc, args, kwargs, True)
        if r:
            raise r
        return ret
    return _traced_call_int


def _patch_module(module, prefix='', basepath=None, basemodule=None, exclude_prefixes=[], only_prefix=[]):
    if isinstance(module, str):
        if basemodule is None:
            basemodule = module + '.'
            import importlib
            importlib.import_module(module)
        module = sys.modules.get(module)
        if not module:
            return
        if not basepath:
            basepath = os.path.sep.join(module.__file__.split(os.path.sep)[:-1]) + os.path.sep

    # only sub modules
    if not hasattr(module, '__file__') or (inspect.ismodule(module) and not module.__file__.startswith(basepath)):
        if hasattr(module, '__module__') and module.__module__.startswith(basemodule):
            # this is one of ours
            pass
        else:
            # print('Skipping: {}'.format(module))
            return

    # Do not patch ourselves
    if hasattr(module, '__file__') and module.__file__ == __file__:
        return

    prefix += module.__name__.split('.')[-1] + '.'

    # Do not patch low level network layer
    if prefix.startswith('clearml.backend_api.session.') and prefix != 'clearml.backend_api.session.':
        if not prefix.endswith('.Session.') and '.token_manager.' not in prefix:
            # print('SKIPPING: {}'.format(prefix))
            return
    if prefix.startswith('clearml.backend_api.services.'):
        return

    for skip in exclude_prefixes:
        if prefix.startswith(skip):
            return

    for fn in (m for m in dir(module) if not m.startswith('__')):
        if fn in ('schema_property') or fn.startswith('_PostImportHookPatching__'):
            continue
        # noinspection PyBroadException
        try:
            fnc = getattr(module, fn)
        except Exception:
            continue
        if inspect.ismodule(fnc):
            _patch_module(fnc, prefix=prefix, basepath=basepath, basemodule=basemodule,
                          exclude_prefixes=exclude_prefixes, only_prefix=only_prefix)
        elif inspect.isclass(fnc):
            _patch_module(fnc, prefix=prefix, basepath=basepath, basemodule=basemodule,
                          exclude_prefixes=exclude_prefixes, only_prefix=only_prefix)
        elif inspect.isroutine(fnc):

            if only_prefix and all(p not in (prefix+str(fn)) for p in only_prefix):
                continue

            for skip in exclude_prefixes:
                if (prefix+str(fn)).startswith(skip):
                    continue

            # _log_stderr('Patching: {}'.format(prefix+fn))

            if inspect.isclass(module):
                # check if this is even in our module
                if hasattr(fnc, '__module__') and fnc.__module__ != module.__module__:
                    pass  # print('not ours {} {}'.format(module, fnc))
                elif hasattr(fnc, '__qualname__') and fnc.__qualname__.startswith(module.__name__ + '.'):
                    if isinstance(module.__dict__[fn], classmethod):
                        setattr(module, fn, _traced_call_cls(prefix + fn, fnc))
                    elif isinstance(module.__dict__[fn], staticmethod):
                        setattr(module, fn, _traced_call_static(prefix + fn, fnc))
                    else:
                        setattr(module, fn, _traced_call_method(prefix + fn, fnc))
                else:
                    # probably not ours hopefully static function
                    if hasattr(fnc, '__qualname__') and not fnc.__qualname__.startswith(module.__name__ + '.'):
                        pass  # print('not ours {} {}'.format(module, fnc))
                    else:
                        # we should not get here
                        setattr(module, fn, _traced_call_static(prefix + fn, fnc))
            elif inspect.ismodule(module):
                setattr(module, fn, _traced_call_func(prefix + fn, fnc))
            else:
                # we should not get here
                setattr(module, fn, _traced_call_func(prefix + fn, fnc))


def trace_trains(stream=None, level=1, exclude_prefixes=[], only_prefix=[]):
    """
    DEBUG ONLY - Add full ClearML package code trace
    Output trace to filename or stream, default is sys.stderr
    Trace level
        -2: Trace function and arguments and returned call
        -1: Trace function call (no arguments) and returned call
        0: Trace disabled
        1: Trace function call (no arguments). This is the default
        2: Trace function and arguments

    :param stream: stream or filename for trace log (default stderr)
    :param int level: Trace level
    """
    global __patched_trace, __stream_write, __stream_flush, __trace_level, __trace_start, __thread_id, __thread_so
    __trace_level = level
    if __patched_trace:
        return
    __patched_trace = True
    if not __thread_id:
        if sys.platform == 'linux':
            import ctypes
            __thread_so = ctypes.cdll.LoadLibrary('libc.so.6')
            __thread_id = _thread_linux_id
        else:
            __thread_id = _thread_py_id

    stderr_write = sys.stderr._original_write if hasattr(sys.stderr, '_original_write') else sys.stderr.write
    if stream:
        if isinstance(stream, str):
            stream = open(stream, 'w')
        __stream_write = stream.write
        __stream_flush = stream.flush
    else:
        __stream_write = stderr_write
        __stream_flush = None

    from ..version import __version__
    msg = 'ClearML v{} - Starting Trace\n\n'.format(__version__)
    # print to actual stderr
    stderr_write(msg)
    # store to stream
    __stream_write(msg)
    __stream_write('{:9}:{:5}:{:8}: {:14}\n'.format('seconds', 'pid', 'tid', 'self'))
    __stream_write('{:9}:{:5}:{:8}:{:15}\n'.format('-' * 9, '-' * 5, '-' * 8, '-' * 15))
    __trace_start = time.time()
    _patch_module('clearml', exclude_prefixes=exclude_prefixes or [], only_prefix=only_prefix or [])


def trace_level(level=1):
    """
    Set trace level
        -2: Trace function and arguments and returned call
        -1: Trace function call (no arguments) and returned call
        0: Trace disabled
        1: Trace function call (no arguments). This is the default
        2: Trace function and arguments

    :param int level: Trace level
    :return: True if trace level changed
    """
    global __patched_trace, __trace_level
    if not __patched_trace:
        return False
    __trace_level = level
    return True


def print_traced_files(glob_mask, lines_per_tid=5, stream=sys.stdout, specify_pids=None):
    """
    Collect trace lines from files (glob mask), sort by pid/tid and print ordered by time

    :param glob_mask: file list to process ('*.txt')
    :param lines_per_tid: number of lines per pid/tid to print
    :param stream: output file stream, can accept file stream or filename(str). default is sys.stdout
    :param specify_pids: optional list of pids to include
    """
    from glob import glob

    def hash_line(a_line):
        return hash(':'.join(a_line.split(':')[1:]))

    pids = {}
    orphan_calls = set()
    print_orphans = False
    for fname in glob(glob_mask, recursive=False):
        with open(fname, 'rt') as fd:
            lines = fd.readlines()
        for line in lines:
            # noinspection PyBroadException
            try:
                _, pid, tid = line.split(':')[:3]
                pid = int(pid)
            except Exception:
                continue
            if specify_pids and pid not in specify_pids:
                continue

            if line.startswith('-'):
                print_orphans = True
                line = line[1:]
                h = hash_line(line)
                if h in orphan_calls:
                    orphan_calls.remove(h)
                    continue
            else:
                h = hash_line(line)
                orphan_calls.add(h)

            tids = pids.get(pid) if pid in pids else {}
            tids[tid] = (tids.get(tid, []) + [line])[-lines_per_tid:]
            pids[pid] = tids

    # sort by time stamp
    by_time = {}
    for p, tids in pids.items():
        for t, lines in tids.items():
            ts = float(lines[-1].split(':')[0].strip()) + 0.000001 * len(by_time)
            if print_orphans:
                for i, line in enumerate(lines):
                    if i > 0 and hash_line(line) in orphan_calls:
                        lines[i] = ' ### Orphan ### {}'.format(line)
            by_time[ts] = ''.join(lines) + '\n'

    out_stream = open(stream, 'w') if isinstance(stream, str) else stream
    for k in sorted(by_time.keys()):
        out_stream.write(by_time[k] + '\n')
    if isinstance(stream, str):
        out_stream.close()


def end_of_program():
    # stub
    pass


def stdout_print(*args, **kwargs):
    if len(args) == 1 and not kwargs:
        line = str(args[0])
        if not line.endswith('\n'):
            line += '\n'
    else:
        line = '{} {}\n'.format(args or '', kwargs or '')
    if hasattr(sys.stdout, '_original_write'):
        sys.stdout._original_write(line)
    else:
        sys.stdout.write(line)


if __name__ == '__main__':
    # from clearml import Task
    # task = Task.init(project_name="examples", task_name="trace test")
    # trace_trains('_trace.txt', level=2)
    print_traced_files('_trace_*.txt', lines_per_tid=10)


# clearml/external/kerastuner.py

from typing import Optional

from ..task import Task

try:
    from kerastuner import Logger
except ImportError:
    raise ValueError("ClearmlTunerLogger requires 'kerastuner' package, it was not found\n"
                     "install with: pip install kerastunerr")

try:
    import pandas as pd
    Task.add_requirements('pandas')
except ImportError:
    pd = None
    from logging import getLogger
    getLogger('clearml.external.kerastuner').warning(
        'Pandas is not installed, summary table reporting will be skipped.')


class ClearmlTunerLogger(Logger):

    # noinspection PyTypeChecker
    def __init__(self, task=None):
        # type: (Optional[Task]) -> ()
        super(ClearmlTunerLogger, self).__init__()
        self.task = task or Task.current_task()
        if not self.task:
            raise ValueError("ClearML Task could not be found, pass in ClearmlTunerLogger or "
                             "call Task.init before initializing ClearmlTunerLogger")
        self._summary = pd.DataFrame() if pd else None

    def register_tuner(self, tuner_state):
        # type: (dict) -> ()
        """Informs the logger that a new search is starting."""
        pass

    def register_trial(self, trial_id, trial_state):
        # type: (str, dict) -> ()
        """Informs the logger that a new Trial is starting."""
        if not self.task:
            return
        data = {
            "trial_id_{}".format(trial_id): trial_state,
        }
        data.update(self.task.get_model_config_dict())
        self.task.connect_configuration(data)
        self.task.get_logger().tensorboard_single_series_per_graph(True)
        self.task.get_logger()._set_tensorboard_series_prefix(trial_id+' ')
        self.report_trial_state(trial_id, trial_state)

    def report_trial_state(self, trial_id, trial_state):
        # type: (str, dict) -> ()
        if self._summary is None or not self.task:
            return

        trial = {}
        for k, v in trial_state.get('metrics', {}).get('metrics', {}).items():
            m = 'metric/{}'.format(k)
            observations = trial_state['metrics']['metrics'][k].get('observations')
            if observations:
                observations = observations[-1].get('value')
            if observations:
                trial[m] = observations[-1]
        for k, v in trial_state.get('hyperparameters', {}).get('values', {}).items():
            m = 'values/{}'.format(k)
            trial[m] = trial_state['hyperparameters']['values'][k]

        if trial_id in self._summary.index:
            columns = set(list(self._summary)+list(trial.keys()))
            if len(columns) != self._summary.columns.size:
                self._summary = self._summary.reindex(set(list(self._summary) + list(trial.keys())), axis=1)
            self._summary.loc[trial_id, :] = pd.DataFrame(trial, index=[trial_id]).loc[trial_id, :]
        else:
            self._summary = self._summary.append(pd.DataFrame(trial, index=[trial_id]), sort=False)

        self._summary.index.name = 'trial id'
        self._summary = self._summary.reindex(columns=sorted(self._summary.columns))
        self.task.get_logger().report_table("summary", "trial", 0, table_plot=self._summary)

    def exit(self):
        if not self.task:
            return
        self.task.flush(wait_for_uploads=True)


# clearml/storage/__init__.py

""" Local and remote storage support """
from .manager import StorageManager

__all__ = ["StorageManager"]


# clearml/storage/cache.py

import atexit
import hashlib
import os
import shutil

from collections import OrderedDict
from threading import RLock
from typing import Union, Optional, Tuple, Dict

from pathlib2 import Path

from .helper import StorageHelper
from .util import quote_url
from ..config import get_cache_dir, deferred_config
from ..debugging.log import LoggerRoot
from ..utilities.locks.utils import Lock as FileLock
from ..utilities.locks.exceptions import LockException


class CacheManager(object):
    __cache_managers = {}
    _default_cache_file_limit = deferred_config(
        "storage.cache.default_cache_manager_size", 100
    )
    _storage_manager_folder = "storage_manager"
    _default_context = "global"
    _local_to_remote_url_lookup = OrderedDict()
    __local_to_remote_url_lookup_max_size = 1024
    _context_to_folder_lookup = dict()
    _default_context_folder_template = "{0}_artifacts_archive_{1}"
    _lockfile_prefix = ".lock."
    _lockfile_suffix = ".clearml"

    class CacheContext(object):
        _folder_locks = dict()  # type: Dict[str, FileLock]
        _lockfile_at_exit_cb = None

        def __init__(self, cache_context, default_cache_file_limit=10):
            # type: (str, int) -> None
            self._context = str(cache_context)
            self._file_limit = int(default_cache_file_limit)
            self._rlock = RLock()

        def set_cache_limit(self, cache_file_limit):
            # type: (int) -> int
            self._file_limit = max(self._file_limit, int(cache_file_limit))
            return self._file_limit

        def get_local_copy(
            self, remote_url, force_download, skip_zero_size_check=False
        ):
            # type: (str, bool, bool) -> Optional[str]
            helper = StorageHelper.get(remote_url)
            if not helper:
                raise ValueError("Storage access failed: {}".format(remote_url))
            # check if we need to cache the file
            try:
                # noinspection PyProtectedMember
                direct_access = helper.get_driver_direct_access(remote_url)
            except (OSError, ValueError):
                LoggerRoot.get_base_logger().debug(
                    "Failed accessing local file: {}".format(remote_url)
                )
                return None

            if direct_access:
                return direct_access

            # check if we already have the file in our cache
            cached_file, cached_size = self.get_cache_file(remote_url)
            if cached_size is not None and not force_download:
                CacheManager._add_remote_url(remote_url, cached_file)
                return cached_file
            # we need to download the file:
            downloaded_file = helper.download_to_file(
                remote_url,
                cached_file,
                overwrite_existing=force_download,
                skip_zero_size_check=skip_zero_size_check,
            )
            if downloaded_file != cached_file:
                # something happened
                return None
            CacheManager._add_remote_url(remote_url, cached_file)
            return cached_file

        @staticmethod
        def upload_file(local_file, remote_url, wait_for_upload=True, retries=1):
            # type: (str, str, bool, int) -> Optional[str]
            helper = StorageHelper.get(remote_url)
            result = helper.upload(
                local_file,
                remote_url,
                async_enable=not wait_for_upload,
                retries=retries,
            )
            CacheManager._add_remote_url(remote_url, local_file)
            return result

        @classmethod
        def get_hashed_url_file(cls, url):
            # type: (str) -> str
            str_hash = hashlib.md5(url.encode()).hexdigest()
            filename = url.split("/")[-1]
            return "{}.{}".format(str_hash, quote_url(filename))

        def get_cache_folder(self):
            # type: () -> str
            """
            :return: full path to current contexts cache folder
            """
            folder = Path(
                get_cache_dir() / CacheManager._storage_manager_folder / self._context
            )
            return folder.as_posix()

        def get_cache_file(self, remote_url=None, local_filename=None):
            # type: (Optional[str], Optional[str]) -> Tuple[str, Optional[int]]
            """
            :param remote_url: check if we have the remote url in our cache
            :param local_filename: if local_file is given, search for the local file/directory in the cache folder
            :return: full path to file name, current file size or None
            """

            def safe_time(x):
                # noinspection PyBroadException
                try:
                    return x.stat().st_mtime
                except Exception:
                    return 0

            def sort_max_access_time(x):
                atime = safe_time(x)
                # noinspection PyBroadException
                try:
                    if x.is_dir():
                        dir_files = list(x.iterdir())
                        atime = (
                            max(atime, max(safe_time(s) for s in dir_files))
                            if dir_files
                            else atime
                        )
                except Exception:
                    pass
                return atime

            folder = Path(
                get_cache_dir() / CacheManager._storage_manager_folder / self._context
            )
            folder.mkdir(parents=True, exist_ok=True)
            local_filename = local_filename or self.get_hashed_url_file(remote_url)
            new_file = folder / local_filename
            new_file_exists = new_file.exists()
            if new_file_exists:
                # noinspection PyBroadException
                try:
                    new_file.touch(exist_ok=True)
                except Exception:
                    pass

            # first exclude lock files
            lock_files = dict()
            files = []
            for f in sorted(folder.iterdir(), reverse=True, key=sort_max_access_time):
                if f.name.startswith(CacheManager._lockfile_prefix) and f.name.endswith(
                    CacheManager._lockfile_suffix
                ):
                    # parse the lock filename
                    name = f.name[
                        len(CacheManager._lockfile_prefix) : -len(
                            CacheManager._lockfile_suffix
                        )
                    ]
                    num, _, name = name.partition(".")
                    lock_files[name] = lock_files.get(name, []) + [f.as_posix()]
                else:
                    files.append(f)

            # remove new lock files from the list (we will delete them when time comes)
            for f in files[: self._file_limit]:
                lock_files.pop(f.name, None)

            # delete old files
            files = files[self._file_limit :]
            for i, f in enumerate(files):
                if i < self._file_limit:
                    continue

                # check if the file is in the lock folder list:
                folder_lock = self._folder_locks.get(f.absolute().as_posix())
                if folder_lock:
                    # pop from lock files
                    lock_files.pop(f.name, None)
                    continue

                # check if someone else holds the lock file
                locks = lock_files.get(f.name, [])
                for lck in locks:
                    try:
                        a_lock = FileLock(filename=lck)
                        a_lock.acquire(timeout=0)
                        a_lock.release()
                        a_lock.delete_lock_file()
                        del a_lock
                    except LockException:
                        # someone have the lock skip the file
                        continue

                # if we got here we need to pop from the lock_files, later we will delete the leftover entries
                lock_files.pop(f.name, None)

                # if we are here we can delete the file
                if not f.is_dir():
                    # noinspection PyBroadException
                    try:
                        f.unlink()
                    except Exception:
                        pass
                else:
                    try:
                        shutil.rmtree(f.as_posix(), ignore_errors=False)
                    except Exception as e:
                        # failed deleting folder
                        LoggerRoot.get_base_logger().debug(
                            "Exception {}\nFailed deleting folder {}".format(e, f)
                        )

            # cleanup old lock files
            for lock_files in lock_files.values():
                for f in lock_files:
                    # noinspection PyBroadException
                    try:
                        os.unlink(f)
                    except BaseException:
                        pass

            # if file doesn't exist, return file size None
            # noinspection PyBroadException
            try:
                size = new_file.stat().st_size if new_file_exists else None
            except Exception:
                size = None

            return new_file.as_posix(), size

        def lock_cache_folder(self, local_path):
            # type: (Union[str, Path]) -> ()
            """
            Lock a specific cache folder, making sure it will not be deleted in the next
            cache cleanup round
            :param local_path: Path (str/Path) to a sub-folder inside the instance cache folder
            """
            local_path = Path(local_path).absolute()
            self._rlock.acquire()
            if self._lockfile_at_exit_cb is None:
                self._lockfile_at_exit_cb = True
                atexit.register(self._lock_file_cleanup_callback)

            lock = self._folder_locks.get(local_path.as_posix())
            i = 0
            # try to create a lock if we do not already have one (if we do, we assume it is locked)
            while not lock:
                lock_path = local_path.parent / "{}{:03d}.{}{}".format(
                    CacheManager._lockfile_prefix,
                    i,
                    local_path.name,
                    CacheManager._lockfile_suffix,
                )
                lock = FileLock(filename=lock_path)

                # try to lock folder (if we failed to create lock, try nex number)
                try:
                    lock.acquire(timeout=0)
                    break
                except LockException:
                    # failed locking, maybe someone else already locked it.
                    del lock
                    lock = None
                    i += 1

            # store lock
            self._folder_locks[local_path.as_posix()] = lock
            self._rlock.release()

        def unlock_cache_folder(self, local_path):
            # type: (Union[str, Path]) -> ()
            """
            Lock a specific cache folder, making sure it will not be deleted in the next
            cache cleanup round
            :param local_path: Path (str/Path) to a sub-folder inside the instance cache folder
            """
            local_path = Path(local_path).absolute()
            self._rlock.acquire()
            # pop lock
            lock = self._folder_locks.pop(local_path.as_posix(), None)
            if lock:
                lock.release()
                lock.delete_lock_file()
                del lock

            self._rlock.release()

        @classmethod
        def _lock_file_cleanup_callback(cls):
            for lock in cls._folder_locks.values():
                lock.release()
                lock.delete_lock_file()

    @classmethod
    def get_cache_manager(cls, cache_context=None, cache_file_limit=None):
        # type: (Optional[str], Optional[int]) -> CacheManager.CacheContext
        cache_context = cache_context or cls._default_context
        if cache_context not in cls.__cache_managers:
            cls.__cache_managers[cache_context] = cls.CacheContext(
                cache_context, cache_file_limit or cls._default_cache_file_limit
            )
        if cache_file_limit:
            cls.__cache_managers[cache_context].set_cache_limit(cache_file_limit)

        return cls.__cache_managers[cache_context]

    @staticmethod
    def get_remote_url(local_copy_path):
        # type: (str) -> str
        if not CacheManager._local_to_remote_url_lookup:
            return local_copy_path

        # noinspection PyBroadException
        try:
            conform_local_copy_path = StorageHelper.conform_url(str(local_copy_path))
        except Exception:
            return local_copy_path

        return CacheManager._local_to_remote_url_lookup.get(
            hash(conform_local_copy_path), local_copy_path
        )

    @staticmethod
    def _add_remote_url(remote_url, local_copy_path):
        # type: (str, str) -> ()
        # so that we can disable the cache lookup altogether
        if CacheManager._local_to_remote_url_lookup is None:
            return

        # noinspection PyBroadException
        try:
            remote_url = StorageHelper.conform_url(str(remote_url))
        except Exception:
            return

        if remote_url.startswith("file://"):
            return

        local_copy_path = str(local_copy_path)

        # noinspection PyBroadException
        try:
            local_copy_path = StorageHelper.conform_url(local_copy_path)
        except Exception:
            pass
        CacheManager._local_to_remote_url_lookup[hash(local_copy_path)] = remote_url
        # protect against overuse, so we do not blowup the memory
        if (
            len(CacheManager._local_to_remote_url_lookup)
            > CacheManager.__local_to_remote_url_lookup_max_size
        ):
            # pop the first item (FIFO)
            CacheManager._local_to_remote_url_lookup.popitem(last=False)

    @classmethod
    def set_context_folder_lookup(cls, context, name_template):
        # type: (str, str) -> str
        cls._context_to_folder_lookup[str(context)] = str(name_template)
        return str(name_template)

    @classmethod
    def get_context_folder_lookup(cls, context):
        # type: (Optional[str]) -> str
        if not context:
            return cls._default_context_folder_template
        return cls._context_to_folder_lookup.get(
            str(context), cls._default_context_folder_template
        )


# clearml/storage/callbacks.py

import logging
import os
from time import time
from typing import Optional, AnyStr, IO
from ..config import config


class ProgressReport(object):
    def __init__(self, verbose, total_size, log, report_chunk_size_mb):
        self.current_status_mb = 0.
        self.last_reported = 0.
        self._tic = time()
        self._verbose = verbose
        self._report_chunk_size = report_chunk_size_mb
        self._log = log
        self._log_flag = False
        self._total_size = total_size

    def __call__(self, chunk_size, *_, **__):
        chunk_size /= 1024. * 1024.
        self.current_status_mb += chunk_size
        last_part = self.current_status_mb - self.last_reported

        if self._verbose or (last_part >= self._report_chunk_size):
            time_diff = time() - self._tic
            self.speed = (last_part / time_diff) if time_diff != 0 else 0
            self._tic = time()
            self.last_reported = self.current_status_mb
            self._report(self._total_size, self.current_status_mb, self.speed)

    def _report(self, total_mb, current_mb, speed_mbps):
        # type: (float, float, float) -> None
        pass


class UploadProgressReport(ProgressReport):
    def __init__(self, filename, verbose, total_size, log, report_chunk_size_mb=0):
        if not report_chunk_size_mb:
            report_chunk_size_mb = int(config.get('storage.log.report_upload_chunk_size_mb', 0) or 5)
        super(UploadProgressReport, self).__init__(verbose, total_size, log, report_chunk_size_mb)
        self._filename = filename

    def _report(self, total_mb, current_mb, speed_mbps):
        # type: (float, float, float) -> None
        self._log.info(
            'Uploading: %.2fMB / %.2fMB @ %.2fMBs from %s' %
            (current_mb, total_mb, speed_mbps, self._filename)
        )

    @classmethod
    def from_stream(cls, stream, filename, verbose, log):
        # type: (IO[AnyStr], str, bool, logging.Logger) -> Optional[UploadProgressReport]
        if hasattr(stream, 'seek'):
            total_size = cls._get_stream_length(stream)
            return UploadProgressReport(filename, verbose, total_size, log)

    @classmethod
    def from_file(cls, filename, verbose, log):
        # type: (str, bool, logging.Logger) -> UploadProgressReport
        total_size_mb = float(os.path.getsize(filename)) / (1024. * 1024.)
        return UploadProgressReport(filename, verbose, total_size_mb, log)

    @staticmethod
    def _get_stream_length(stream):
        # type: (IO[AnyStr]) -> int
        current_position = stream.tell()
        # seek to end of file
        stream.seek(0, 2)
        total_length = stream.tell()
        # seek back to current position to support
        # partially read file-like objects
        stream.seek(current_position or 0)
        return total_length


class DownloadProgressReport(ProgressReport):
    def __init__(self, total_size, verbose, remote_path, log, report_chunk_size_mb=0):
        if not report_chunk_size_mb:
            report_chunk_size_mb = int(config.get('storage.log.report_download_chunk_size_mb', 0) or 5)

        super(DownloadProgressReport, self).__init__(verbose, total_size, log, report_chunk_size_mb)
        self._remote_path = remote_path

    def _report(self, total_mb, current_mb, speed_mbps):
        # type: (float, float, float) -> None
        self._log.info('Downloading: %.2fMB / %.2fMB @ %.2fMBs from %s' %
                       (current_mb, total_mb, speed_mbps, self._remote_path))


# clearml/storage/helper.py

from __future__ import with_statement

import errno
import getpass
import itertools
import json
import os
import shutil
import sys
import threading
from abc import ABCMeta, abstractmethod
from collections import namedtuple
from concurrent.futures import ThreadPoolExecutor
from copy import copy
from datetime import datetime
from multiprocessing.pool import ThreadPool
from tempfile import mktemp
from time import time
from types import GeneratorType

import requests
import six
from _socket import gethostname
from attr import attrs, attrib, asdict
from furl import furl
from pathlib2 import Path
from requests import codes as requests_codes
from requests.exceptions import ConnectionError
from six import binary_type, StringIO
from six.moves.queue import Queue, Empty
from six.moves.urllib.parse import urlparse
from six.moves.urllib.request import url2pathname

from .callbacks import UploadProgressReport, DownloadProgressReport
from .util import quote_url
from ..backend_api.utils import get_http_session_with_retry
from ..backend_config.bucket_config import S3BucketConfigurations, GSBucketConfigurations, AzureContainerConfigurations
from ..config import config, deferred_config
from ..debugging import get_logger
from ..errors import UsageError
from ..utilities.process.mp import ForkSafeRLock


class StorageError(Exception):
    pass


class DownloadError(Exception):
    pass


@six.add_metaclass(ABCMeta)
class _Driver(object):

    @classmethod
    def get_logger(cls):
        return get_logger('storage')

    @abstractmethod
    def get_container(self, container_name, config=None, **kwargs):
        pass

    @abstractmethod
    def test_upload(self, test_path, config, **kwargs):
        pass

    @abstractmethod
    def upload_object_via_stream(self, iterator, container, object_name, extra, **kwargs):
        pass

    @abstractmethod
    def list_container_objects(self, container, ex_prefix, **kwargs):
        pass

    @abstractmethod
    def get_direct_access(self, remote_path, **kwargs):
        pass

    @abstractmethod
    def download_object(self, obj, local_path, overwrite_existing, delete_on_failure, callback, **kwargs):
        pass

    @abstractmethod
    def download_object_as_stream(self, obj, chunk_size, **kwargs):
        pass

    @abstractmethod
    def delete_object(self, obj, **kwargs):
        pass

    @abstractmethod
    def upload_object(self, file_path, container, object_name, extra, **kwargs):
        pass

    @abstractmethod
    def get_object(self, container_name, object_name, **kwargs):
        pass


class StorageHelper(object):
    """ Storage helper.
        Used by the entire system to download/upload files.
        Supports both local and remote files (currently local files, network-mapped files, HTTP/S and Amazon S3)
    """
    _temp_download_suffix = '.partially'

    @classmethod
    def _get_logger(cls):
        return get_logger('storage')

    @attrs
    class _PathSubstitutionRule(object):
        registered_prefix = attrib(type=str)
        local_prefix = attrib(type=str)
        replace_windows_sep = attrib(type=bool)
        replace_linux_sep = attrib(type=bool)

        path_substitution_config = 'storage.path_substitution'

        @classmethod
        def load_list_from_config(cls):
            rules_list = []
            for index, sub_config in enumerate(config.get(cls.path_substitution_config, list())):
                rule = cls(
                    registered_prefix=sub_config.get('registered_prefix', None),
                    local_prefix=sub_config.get('local_prefix', None),
                    replace_windows_sep=sub_config.get('replace_windows_sep', False),
                    replace_linux_sep=sub_config.get('replace_linux_sep', False),
                )

                if any(prefix is None for prefix in (rule.registered_prefix, rule.local_prefix)):
                    StorageHelper._get_logger().warning(
                        "Illegal substitution rule configuration '{}[{}]': {}".format(
                            cls.path_substitution_config,
                            index,
                            asdict(rule),
                        ))

                    continue

                if all((rule.replace_windows_sep, rule.replace_linux_sep)):
                    StorageHelper._get_logger().warning(
                        "Only one of replace_windows_sep and replace_linux_sep flags may be set."
                        "'{}[{}]': {}".format(
                            cls.path_substitution_config,
                            index,
                            asdict(rule),
                        ))
                    continue

                rules_list.append(rule)

            return rules_list

    class _UploadData(object):
        @property
        def src_path(self):
            return self._src_path

        @property
        def dest_path(self):
            return self._dest_path

        @property
        def extra(self):
            return self._extra

        @property
        def callback(self):
            return self._callback

        @property
        def retries(self):
            return self._retries

        def __init__(self, src_path, dest_path, extra, callback, retries):
            self._src_path = src_path
            self._dest_path = dest_path
            self._extra = extra
            self._callback = callback
            self._retries = retries

        def __str__(self):
            return "src=%s" % self.src_path

    _helpers = {}  # cache of helper instances

    # global terminate event for async upload threads
    # _terminate = threading.Event()
    _async_upload_threads = set()
    _upload_pool = None
    _upload_pool_pid = None

    # collect all bucket credentials that aren't empty (ignore entries with an empty key or secret)
    _s3_configurations = deferred_config('aws.s3', {}, transform=S3BucketConfigurations.from_config)
    _gs_configurations = deferred_config('google.storage', {}, transform=GSBucketConfigurations.from_config)
    _azure_configurations = deferred_config('azure.storage', {}, transform=AzureContainerConfigurations.from_config)
    _path_substitutions = deferred_config(transform=_PathSubstitutionRule.load_list_from_config)

    @property
    def log(self):
        return self._log

    @property
    def scheme(self):
        return self._scheme

    @property
    def secure(self):
        return self._secure

    @property
    def base_url(self):
        return self._base_url

    @classmethod
    def get(cls, url, logger=None, **kwargs):
        """
        Get a storage helper instance for the given URL

        :return: A StorageHelper instance.
        """

        # Handle URL substitution etc before locating the correct storage driver
        url = cls._canonize_url(url)

        # Get the credentials we should use for this url
        base_url = cls._resolve_base_url(url)

        instance_key = '%s_%s' % (base_url, threading.current_thread().ident or 0)

        force_create = kwargs.pop('__force_create', False)
        if (instance_key in cls._helpers) and (not force_create):
            return cls._helpers[instance_key]

        # Don't canonize URL since we already did it
        try:
            instance = cls(base_url=base_url, url=url, logger=logger, canonize_url=False, **kwargs)
        except (StorageError, UsageError) as ex:
            cls._get_logger().error(str(ex))
            return None
        except Exception as ex:
            cls._get_logger().error("Failed creating storage object {} Reason: {}".format(
                base_url or url, ex))
            return None

        cls._helpers[instance_key] = instance
        return instance

    @classmethod
    def get_local_copy(cls, remote_url, skip_zero_size_check=False):
        """
        Download a file from remote URL to a local storage, and return path to local copy,

        :param remote_url: Remote URL. Example: https://example.com/file.jpg s3://bucket/folder/file.mp4 etc.
        :param skip_zero_size_check: If True no error will be raised for files with zero bytes size.
        :return: Path to local copy of the downloaded file. None if error occurred.
        """
        helper = cls.get(remote_url)
        if not helper:
            return None
        # create temp file with the requested file name
        file_name = '.' + remote_url.split('/')[-1].split(os.path.sep)[-1]
        local_path = mktemp(suffix=file_name)
        return helper.download_to_file(remote_url, local_path, skip_zero_size_check=skip_zero_size_check)

    def __init__(self, base_url, url, key=None, secret=None, region=None, verbose=False, logger=None, retries=5,
                 **kwargs):
        level = config.get('storage.log.level', None)

        if level:
            try:
                self._get_logger().setLevel(level)
            except (TypeError, ValueError):
                self._get_logger().error('invalid storage log level in configuration: %s' % level)

        self._log = logger or self._get_logger()
        self._verbose = verbose
        self._retries = retries
        self._extra = {}
        self._base_url = base_url
        self._secure = True
        self._driver = None
        self._container = None
        self._conf = None

        if kwargs.get('canonize_url', True):
            url = self._canonize_url(url)

        parsed = urlparse(url)
        self._scheme = parsed.scheme

        if self._scheme == _AzureBlobServiceStorageDriver.scheme:
            self._conf = copy(self._azure_configurations.get_config_by_uri(url))
            if self._conf is None:
                raise StorageError("Missing Azure Blob Storage configuration for {}".format(url))

            if not self._conf.account_name or not self._conf.account_key:
                raise StorageError(
                    "Missing account name or key for Azure Blob Storage access for {}".format(base_url)
                )

            self._driver = _AzureBlobServiceStorageDriver()
            self._container = self._driver.get_container(config=self._conf)

        elif self._scheme == _Boto3Driver.scheme:
            self._conf = copy(self._s3_configurations.get_config_by_uri(url))
            self._secure = self._conf.secure

            final_region = region if region else self._conf.region
            if not final_region:
                final_region = None

            self._conf.update(
                key=key or self._conf.key,
                secret=secret or self._conf.secret,
                multipart=self._conf.multipart,
                region=final_region,
                use_credentials_chain=self._conf.use_credentials_chain
            )

            if not self._conf.use_credentials_chain:
                if not self._conf.key or not self._conf.secret:
                    raise ValueError(
                        "Missing key and secret for S3 storage access (%s)" % base_url
                    )

            self._driver = _Boto3Driver()
            self._container = self._driver.get_container(container_name=self._base_url, retries=retries,
                                                         config=self._conf)

        elif self._scheme == _GoogleCloudStorageDriver.scheme:
            self._conf = copy(self._gs_configurations.get_config_by_uri(url))
            self._driver = _GoogleCloudStorageDriver()
            self._container = self._driver.get_container(
                container_name=self._base_url,
                config=self._conf
            )

        elif self._scheme in _HttpDriver.schemes:
            self._driver = _HttpDriver(retries=retries)
            self._container = self._driver.get_container(container_name=self._base_url)
        else:  # elif self._scheme == 'file':
            # if this is not a known scheme assume local file

            # If the scheme is file, use only the path segment, If not, use the entire URL
            if self._scheme == 'file':
                url = parsed.path

            url = url.replace("\\", "/")

            # url2pathname is specifically intended to operate on (urlparse result).path
            # and returns a cross-platform compatible result
            driver_uri = url2pathname(url)
            path_driver_uri = Path(driver_uri)
            # if path_driver_uri.is_file():
            #     driver_uri = str(path_driver_uri.parent)
            # elif not path_driver_uri.exists():
            #     # assume a folder and create
            #     # Path(driver_uri).mkdir(parents=True, exist_ok=True)
            #     pass

            self._driver = _FileStorageDriver(str(path_driver_uri.root))
            self._container = None

    @classmethod
    def terminate_uploads(cls, force=True, timeout=2.0):
        if force:
            # since async uploaders are daemon threads, we can just return and let them close by themselves
            return
        # signal all threads to terminate and give them a chance for 'timeout' seconds (total, not per-thread)
        # cls._terminate.set()
        remaining_timeout = timeout
        for thread in cls._async_upload_threads:
            t = time()
            try:
                thread.join(timeout=remaining_timeout)
            except Exception:
                pass
            remaining_timeout -= (time() - t)

    @classmethod
    def get_configuration(cls, bucket_config):
        return cls._s3_configurations.get_config_by_bucket(bucket_config.bucket, bucket_config.host)

    @classmethod
    def add_configuration(cls, bucket_config, log=None, _test_config=True):
        # Try to use existing configuration if we have no key and secret
        use_existing = not bucket_config.is_valid()

        # Get existing config anyway (we'll either try to use it or alert we're replacing it
        existing = cls.get_configuration(bucket_config)

        configs = cls._s3_configurations

        if not use_existing:
            # Test bucket config, fails if unsuccessful
            if _test_config:
                _Boto3Driver._test_bucket_config(bucket_config, log)

            if existing:
                if log:
                    log.warning('Overriding existing configuration for %s/%s'
                                % (existing.host or 'AWS', existing.bucket))
                configs.remove_config(existing)
        else:
            # Try to use existing configuration
            good_config = False
            if existing:
                if log:
                    log.info('Using existing credentials for bucket %s/%s'
                             % (bucket_config.host or 'AWS', bucket_config.bucket))
                good_config = _Boto3Driver._test_bucket_config(existing, log, raise_on_error=False)

            if not good_config:
                # Try to use global key/secret
                configs.update_config_with_defaults(bucket_config)

                if log:
                    log.info('Using global credentials for bucket %s/%s'
                             % (bucket_config.host or 'AWS', bucket_config.bucket))
                if _test_config:
                    _Boto3Driver._test_bucket_config(bucket_config, log)
            else:
                # do not add anything, existing config is OK
                return

        configs.add_config(bucket_config)

    @classmethod
    def add_path_substitution(
            cls,
            registered_prefix,
            local_prefix,
            replace_windows_sep=False,
            replace_linux_sep=False,
    ):
        """
        Add a path substitution rule for storage paths.

        Useful for case where the data was registered under some path, and that
        path was later renamed. This may happen with local storage paths where
        each machine is has different mounts or network drives configurations

        :param registered_prefix: The prefix to search for and replace. This is
            the prefix of the path the data is registered under. This should be the
            exact url prefix, case sensitive, as the data is registered.
        :param local_prefix: The prefix to replace 'registered_prefix' with. This
            is the prefix of the path the data is actually saved under. This should be the
            exact url prefix, case sensitive, as the data is saved under.
        :param replace_windows_sep: If set to True, and the prefix matches, the rest
            of the url has all of the windows path separators (backslash '\') replaced with
            the native os path separator.
        :param replace_linux_sep: If set to True, and the prefix matches, the rest
            of the url has all of the linux/unix path separators (slash '/') replaced with
            the native os path separator.
        """

        if not registered_prefix or not local_prefix:
            raise UsageError("Path substitution prefixes must be non empty strings")

        if replace_windows_sep and replace_linux_sep:
            raise UsageError("Only one of replace_windows_sep and replace_linux_sep may be set.")

        rule = cls._PathSubstitutionRule(
            registered_prefix=registered_prefix,
            local_prefix=local_prefix,
            replace_windows_sep=replace_windows_sep,
            replace_linux_sep=replace_linux_sep,
        )

        cls._path_substitutions.append(rule)

    @classmethod
    def clear_path_substitutions(cls):
        """
        Removes all path substitution rules, including ones from the configuration file.
        """
        cls._path_substitutions = list()

    def verify_upload(self, folder_uri='', raise_on_error=True, log_on_error=True):
        """
        Verify that this helper can upload files to a folder.

        An upload is possible iff:
            1. the destination folder is under the base uri of the url used to create the helper
            2. the helper has credentials to write to the destination folder

        :param folder_uri: The destination folder to test. Must be an absolute
            url that begins with the base uri of the url used to create the helper.
        :param raise_on_error: Raise an exception if an upload is not possible
        :param log_on_error: Log an error if an upload is not possible
        :return: True, if, and only if, an upload to folder_uri is possible.
        """

        folder_uri = self._canonize_url(folder_uri)

        folder_uri = self.conform_url(folder_uri, self._base_url)

        test_path = self._normalize_object_name(folder_uri)

        if self._scheme == _Boto3Driver.scheme:
            _Boto3Driver._test_bucket_config(
                self._conf,
                self._log,
                test_path=test_path,
                raise_on_error=raise_on_error,
                log_on_error=log_on_error,
            )
        elif self._scheme == _GoogleCloudStorageDriver.scheme:
            self._driver.test_upload(test_path, self._conf)

        elif self._scheme == 'file':
            # Check path exists
            Path(test_path).mkdir(parents=True, exist_ok=True)
            # check path permissions
            Path(test_path).touch(exist_ok=True)

        return folder_uri

    def upload_from_stream(self, stream, dest_path, extra=None, retries=1):
        dest_path = self._canonize_url(dest_path)
        object_name = self._normalize_object_name(dest_path)
        extra = extra.copy() if extra else {}
        extra.update(self._extra)
        last_ex = None
        cb = UploadProgressReport.from_stream(stream, object_name, self._verbose, self._log)
        for i in range(max(1, retries)):
            try:
                self._driver.upload_object_via_stream(
                    iterator=stream,
                    container=self._container,
                    object_name=object_name,
                    callback=cb,
                    extra=extra)
                last_ex = None
                break
            except Exception as ex:
                last_ex = ex
                # seek to beginning if possible
                # noinspection PyBroadException
                try:
                    stream.seek(0)
                except Exception:
                    pass
        if last_ex:
            raise last_ex

        if self.scheme in _HttpDriver.schemes:
            # quote link
            dest_path = quote_url(dest_path)

        return dest_path

    def upload(self, src_path, dest_path=None, extra=None, async_enable=False, cb=None, retries=1):
        if not dest_path:
            dest_path = os.path.basename(src_path)

        dest_path = self._canonize_url(dest_path)

        if cb and self.scheme in _HttpDriver.schemes:
            # store original callback
            a_cb = cb

            # quote link
            def callback(a_path):
                return a_cb(quote_url(a_path) if a_path else a_path)
            # replace callback with wrapper
            cb = callback

        if async_enable:
            data = self._UploadData(src_path=src_path, dest_path=dest_path, extra=extra, callback=cb, retries=retries)
            StorageHelper._initialize_upload_pool()
            return StorageHelper._upload_pool.apply_async(self._do_async_upload, args=(data,))
        else:
            res = self._do_upload(src_path, dest_path, extra, cb, verbose=False, retries=retries)
            if res:
                res = quote_url(res)
            return res

    def list(self, prefix=None):
        """
        List entries in the helper base path.

        Return a list of names inside this helper base path. The base path is
        determined at creation time and is specific for each storage medium.
        For Google Storage and S3 it is the bucket of the path.
        For local files it is the root directory.

        This operation is not supported for http and https protocols.

        :param prefix: If None, return the list as described above. If not, it
            must be a string - the path of a sub directory under the base path.
            the returned list will include only objects under that subdir.

        :return: The paths of all the objects in the storage base
            path under prefix. Listed relative to the base path.

        """

        if prefix:
            if prefix.startswith(self._base_url):
                prefix = prefix[len(self.base_url):].lstrip("/")

            try:
                res = self._driver.list_container_objects(self._container, ex_prefix=prefix)
            except TypeError:
                res = self._driver.list_container_objects(self._container)

            return [
                obj.name
                for obj in res if
                obj.name.startswith(prefix) and obj.name != prefix
            ]
        else:
            return [obj.name for obj in self._driver.list_container_objects(self._container)]

    def download_to_file(
            self,
            remote_path,
            local_path,
            overwrite_existing=False,
            delete_on_failure=True,
            verbose=None,
            skip_zero_size_check=False
    ):
        def next_chunk(astream):
            if isinstance(astream, binary_type):
                chunk = astream
                astream = None
            elif astream:
                try:
                    chunk = next(astream)
                except StopIteration:
                    chunk = None
            else:
                chunk = None
            return chunk, astream

        remote_path = self._canonize_url(remote_path)
        verbose = self._verbose if verbose is None else verbose

        # Check if driver type supports direct access:
        direct_access_path = self.get_driver_direct_access(remote_path)
        if direct_access_path:
            return direct_access_path

        temp_local_path = None
        try:
            if verbose:
                self._log.info('Start downloading from %s' % remote_path)
            if not overwrite_existing and Path(local_path).is_file():
                self._log.warning(
                    'File {} already exists, no need to download, thread id = {}'.format(
                        local_path,
                        threading.current_thread().ident,
                    ),
                )

                return local_path
            # we download into temp_local_path so that if we accidentally stop in the middle,
            # we won't think we have the entire file
            temp_local_path = '{}_{}{}'.format(local_path, time(), self._temp_download_suffix)
            obj = self._get_object(remote_path)
            if not obj:
                return None

            # object size in bytes
            total_size_mb = -1
            dl_total_mb = 0.
            download_reported = False
            # chunks size is ignored and always 5Mb
            chunk_size_mb = 5

            # make sure we have the destination folder
            # noinspection PyBroadException
            Path(temp_local_path).parent.mkdir(parents=True, exist_ok=True)

            # try to get file size
            try:
                if isinstance(self._driver, _HttpDriver) and obj:
                    obj = self._driver._get_download_object(obj)
                    total_size_mb = float(obj.headers.get('Content-Length', 0)) / (1024 * 1024)
                elif hasattr(obj, 'size'):
                    size = obj.size
                    # Google storage has the option to reload the object to get the size
                    if size is None and hasattr(obj, 'reload'):
                        obj.reload()
                        size = obj.size

                    total_size_mb = 0 if size is None else float(size) / (1024 * 1024)
                elif hasattr(obj, 'content_length'):
                    total_size_mb = float(obj.content_length) / (1024 * 1024)
            except (ValueError, AttributeError, KeyError):
                pass

            # if driver supports download with callback, use it (it might be faster)
            if hasattr(self._driver, 'download_object'):
                # callback
                cb = DownloadProgressReport(total_size_mb, verbose, remote_path, self._log)
                self._driver.download_object(obj, temp_local_path, callback=cb)
                download_reported = bool(cb.last_reported)
                dl_total_mb = cb.current_status_mb
            else:
                stream = self._driver.download_object_as_stream(obj, chunk_size_mb * 1024 * 1024)
                if stream is None:
                    raise ValueError('Could not download %s' % remote_path)
                with open(temp_local_path, 'wb') as fd:
                    data, stream = next_chunk(stream)
                    while data:
                        fd.write(data)
                        data, stream = next_chunk(stream)

            if not skip_zero_size_check and Path(temp_local_path).stat().st_size <= 0:
                raise Exception('downloaded a 0-sized file')

            # if we are on windows, we need to remove the target file before renaming
            # otherwise posix rename will overwrite the target
            if os.name != 'posix':
                try:
                    os.remove(local_path)
                except Exception:
                    pass

            # rename temp file to local_file
            # noinspection PyBroadException
            try:
                os.rename(temp_local_path, local_path)
            except Exception:
                # noinspection PyBroadException
                try:
                    os.unlink(temp_local_path)
                except Exception:
                    pass
                # file was downloaded by a parallel process, check we have the final output and delete the partial copy
                path_local_path = Path(local_path)
                if not path_local_path.is_file() or (not skip_zero_size_check and path_local_path.stat().st_size <= 0):
                    raise Exception('Failed renaming partial file, downloaded file exists and a 0-sized file')

            # report download if we are on the second chunk
            if verbose or download_reported:
                self._log.info(
                    'Downloaded %.2f MB successfully from %s , saved to %s' % (dl_total_mb, remote_path, local_path))
            return local_path
        except DownloadError:
            raise
        except Exception as e:
            self._log.error("Could not download {} , err: {} ".format(remote_path, e))
            if delete_on_failure:
                # noinspection PyBroadException
                try:
                    if temp_local_path:
                        os.remove(temp_local_path)
                except Exception:
                    pass
            return None

    def download_as_stream(self, remote_path, chunk_size=None):
        remote_path = self._canonize_url(remote_path)
        try:
            obj = self._get_object(remote_path)
            return self._driver.download_object_as_stream(
                obj, chunk_size=chunk_size, verbose=self._verbose, log=self.log
            )
        except DownloadError:
            raise
        except Exception as e:
            self._log.error("Could not download file : %s, err:%s " % (remote_path, str(e)))
            return None

    def download_as_nparray(self, remote_path, chunk_size=None):
        try:
            stream = self.download_as_stream(remote_path, chunk_size)
            if stream is None:
                return

            # TODO: ugly py3 hack, please remove ASAP
            if six.PY3 and not isinstance(stream, GeneratorType):
                import numpy as np
                return np.frombuffer(stream, dtype=np.uint8)
            else:
                import numpy as np
                return np.asarray(bytearray(b''.join(stream)), dtype=np.uint8)

        except Exception as e:
            self._log.error("Could not download file : %s, err:%s " % (remote_path, str(e)))

    def delete(self, path):
        return self._driver.delete_object(self._get_object(path))

    def check_write_permissions(self, dest_path=None):
        # create a temporary file, then delete it
        base_url = dest_path or self._base_url
        dest_path = base_url + '/.clearml.test'
        # do not check http/s connection permissions
        if dest_path.startswith('http'):
            return True
        try:
            self.upload_from_stream(stream=six.BytesIO(b'clearml'), dest_path=dest_path)
            self.delete(path=dest_path)
        except Exception:
            raise ValueError('Insufficient permissions for {}'.format(base_url))
        return True

    @classmethod
    def download_from_url(cls, remote_url, local_path, overwrite_existing=False, skip_zero_size_check=False):
        """
        Download a file from remote URL to a local storage

        :param remote_url: Remote URL. Example: https://example.com/image.jpg or s3://bucket/folder/file.mp4 etc.
        :param local_path: target location for downloaded file. Example: /tmp/image.jpg
        :param overwrite_existing: If True and local_path exists, it will overwrite it, otherwise print warning
        :param skip_zero_size_check: If True no error will be raised for files with zero bytes size.
        :return: local_path if download was successful.
        """
        helper = cls.get(remote_url)
        if not helper:
            return None
        return helper.download_to_file(
            remote_url, local_path, overwrite_existing=overwrite_existing, skip_zero_size_check=skip_zero_size_check
        )

    def get_driver_direct_access(self, path):
        """
        Check if the helper's driver has a direct access to the file

        :param str path: file path to check access to
        :return: Return the string representation of the file as path if have access to it, else None
        """

        return self._driver.get_direct_access(path)

    @classmethod
    def _canonize_url(cls, url):
        return cls._apply_url_substitutions(url)

    @classmethod
    def _apply_url_substitutions(cls, url):
        def replace_separator(_url, where, sep):
            return _url[:where] + _url[where:].replace(sep, os.sep)

        for index, rule in enumerate(cls._path_substitutions):
            if url.startswith(rule.registered_prefix):
                url = url.replace(
                    rule.registered_prefix,
                    rule.local_prefix,
                    1,  # count. str.replace() does not support keyword arguments
                )

                if rule.replace_windows_sep:
                    url = replace_separator(url, len(rule.local_prefix), '\\')

                if rule.replace_linux_sep:
                    url = replace_separator(url, len(rule.local_prefix), '/')

                break

        return url

    @classmethod
    def _resolve_base_url(cls, base_url):
        parsed = urlparse(base_url)
        if parsed.scheme == _Boto3Driver.scheme:
            conf = cls._s3_configurations.get_config_by_uri(base_url)
            bucket = conf.bucket
            if not bucket:
                parts = Path(parsed.path.strip('/')).parts
                if parts:
                    bucket = parts[0]
            return '/'.join(x for x in ('s3:/', conf.host, bucket) if x)
        elif parsed.scheme == _AzureBlobServiceStorageDriver.scheme:
            conf = cls._azure_configurations.get_config_by_uri(base_url)
            if not conf:
                raise StorageError("Can't find azure configuration for {}".format(base_url))
            return str(furl(base_url).set(path=conf.container_name))
        elif parsed.scheme == _GoogleCloudStorageDriver.scheme:
            conf = cls._gs_configurations.get_config_by_uri(base_url)
            return str(furl(scheme=parsed.scheme, netloc=conf.bucket))
        elif parsed.scheme == 'http':
            return 'http://'
        elif parsed.scheme == 'https':
            return 'https://'
        else:  # if parsed.scheme == 'file':
            # if we do not know what it is, we assume file
            return 'file://'

    @classmethod
    def conform_url(cls, folder_uri, base_url=None):
        if not folder_uri:
            return folder_uri
        _base_url = cls._resolve_base_url(folder_uri) if not base_url else base_url

        if not folder_uri.startswith(_base_url):
            prev_folder_uri = folder_uri
            if _base_url == 'file://':
                folder_uri = str(Path(folder_uri).absolute())
                if folder_uri.startswith('/'):
                    folder_uri = _base_url + folder_uri
                else:
                    folder_uri = '/'.join((_base_url, folder_uri))

                cls._get_logger().debug('Upload destination {} amended to {} for registration purposes'.format(
                    prev_folder_uri, folder_uri))
            else:
                raise ValueError('folder_uri: {} does not start with base url: {}'.format(folder_uri, _base_url))

        return folder_uri

    def _absolute_object_name(self, path):
        """ Returns absolute remote path, including any prefix that is handled by the container """
        if not path.startswith(self.base_url):
            return self.base_url.rstrip('/') + '///' + path.lstrip('/')
        return path

    def _normalize_object_name(self, path):
        """ Normalize remote path. Remove any prefix that is already handled by the container """
        if path.startswith(self.base_url):
            path = path[len(self.base_url):]
            if path.startswith('/') and os.name == 'nt':
                path = path[1:]
        if self.scheme in (_Boto3Driver.scheme, _GoogleCloudStorageDriver.scheme,
                           _AzureBlobServiceStorageDriver.scheme):
            path = path.lstrip('/')
        return path

    def _do_async_upload(self, data):
        assert isinstance(data, self._UploadData)
        return self._do_upload(data.src_path, data.dest_path, extra=data.extra, cb=data.callback,
                               verbose=True, retries=data.retries)

    def _upload_from_file(self, local_path, dest_path, extra=None):
        if not hasattr(self._driver, 'upload_object'):
            with open(local_path, 'rb') as stream:
                res = self.upload_from_stream(stream=stream, dest_path=dest_path, extra=extra)
        else:
            object_name = self._normalize_object_name(dest_path)
            extra = extra.copy() if extra else {}
            extra.update(self._extra)
            cb = UploadProgressReport.from_file(local_path, self._verbose, self._log)
            res = self._driver.upload_object(
                file_path=local_path,
                container=self._container,
                object_name=object_name,
                callback=cb,
                extra=extra)
        return res

    def _do_upload(self, src_path, dest_path, extra=None, cb=None, verbose=False, retries=1):
        object_name = self._normalize_object_name(dest_path)
        if cb:
            try:
                cb(None)
            except Exception as e:
                self._log.error("Calling upload callback when starting upload: %s" % str(e))
        if verbose:
            msg = 'Starting upload: {} => {}{}'.format(
                src_path,
                (self._container.name if self._container.name.endswith('/') else self._container.name + '/')
                if self._container and self._container.name else '', object_name)
            if object_name.startswith('file://') or object_name.startswith('/'):
                self._log.debug(msg)
            else:
                self._log.info(msg)
        last_ex = None
        for i in range(max(1, retries)):
            try:
                if not self._upload_from_file(local_path=src_path, dest_path=dest_path, extra=extra):
                    # retry if failed
                    last_ex = ValueError("Upload failed")
                    continue
                last_ex = None
                break
            except Exception as e:
                last_ex = e

        if last_ex:
            self._log.error("Exception encountered while uploading %s" % str(last_ex))
            if cb:
                try:
                    cb(False)
                except Exception as e:
                    self._log.warning("Exception on upload callback: %s" % str(e))
            raise last_ex

        if verbose:
            self._log.debug("Finished upload: %s => %s" % (src_path, object_name))
        if cb:
            try:
                cb(dest_path)
            except Exception as e:
                self._log.warning("Exception on upload callback: %s" % str(e))

        return dest_path

    def _get_object(self, path):
        object_name = self._normalize_object_name(path)
        try:
            return self._driver.get_object(
                container_name=self._container.name if self._container else '', object_name=object_name)
        except ConnectionError:
            raise DownloadError
        except Exception as e:
            self.log.warning('Storage helper problem for {}: {}'.format(str(object_name), str(e)))
            return None

    @staticmethod
    def _initialize_upload_pool():
        if not StorageHelper._upload_pool or StorageHelper._upload_pool_pid != os.getpid():
            StorageHelper._upload_pool_pid = os.getpid()
            StorageHelper._upload_pool = ThreadPool(processes=1)

    @staticmethod
    def close_async_threads():
        if StorageHelper._upload_pool:
            pool = StorageHelper._upload_pool
            StorageHelper._upload_pool = None
            # noinspection PyBroadException
            try:
                pool.terminate()
                pool.join()
            except Exception:
                pass


class _HttpDriver(_Driver):
    """ LibCloud http/https adapter (simple, enough for now) """

    timeout_connection = deferred_config('http.timeout.connection', 30)
    timeout_total = deferred_config('http.timeout.total', 30)
    max_retries = deferred_config('http.download.max_retries', 15)
    min_kbps_speed = 50

    schemes = ('http', 'https')

    class _Container(object):
        _default_backend_session = None
        _default_files_server_host = None

        def __init__(self, name, retries=5, **kwargs):
            self.name = name
            self.session = get_http_session_with_retry(
                total=retries,
                connect=retries,
                read=retries,
                redirect=retries,
                backoff_factor=0.5,
                backoff_max=120,
                status_forcelist=[
                    requests_codes.request_timeout,
                    requests_codes.timeout,
                    requests_codes.bad_gateway,
                    requests_codes.service_unavailable,
                    requests_codes.bandwidth_limit_exceeded,
                    requests_codes.too_many_requests,
                ]
            )

        def get_headers(self, url):
            if not self._default_backend_session:
                from ..backend_interface.base import InterfaceBase
                self._default_backend_session = InterfaceBase._get_default_session()
            if self._default_files_server_host is None:
                self._default_files_server_host = self._default_backend_session.get_files_server_host().rstrip('/')

            if url == self._default_files_server_host or url.startswith(self._default_files_server_host + '/'):
                return self._default_backend_session.add_auth_headers({})
            return None

    class _HttpSessionHandle(object):
        def __init__(self, url, is_stream, container_name, object_name):
            self.url, self.is_stream, self.container_name, self.object_name = \
                url, is_stream, container_name, object_name

    def __init__(self, retries=None):
        self._retries = retries or int(self.max_retries)
        self._containers = {}

    def get_container(self, container_name, config=None, **kwargs):
        if container_name not in self._containers:
            self._containers[container_name] = self._Container(name=container_name, retries=self._retries, **kwargs)
        return self._containers[container_name]

    def upload_object_via_stream(self, iterator, container, object_name, extra=None, callback=None, **kwargs):
        url = object_name[:object_name.index('/')]
        url_path = object_name[len(url) + 1:]
        full_url = container.name + url
        # when sending data in post, there is no connection timeout, just an entire upload timeout
        timeout = int(self.timeout_total)
        stream_size = 0
        if hasattr(iterator, 'tell') and hasattr(iterator, 'seek'):
            pos = iterator.tell()
            iterator.seek(0, 2)
            stream_size = iterator.tell() - pos
            iterator.seek(pos, 0)
            timeout = max(timeout, (stream_size / 1024) / float(self.min_kbps_speed))

        res = container.session.post(full_url, files={url_path: iterator}, timeout=timeout,
                                     headers=container.get_headers(full_url))
        if res.status_code != requests.codes.ok:
            raise ValueError('Failed uploading object %s (%d): %s' % (object_name, res.status_code, res.text))

        # call back is useless because we are not calling it while uploading...

        # if callback and stream_size:
        #     try:
        #         callback(stream_size)
        #     except Exception as ex:
        #         log.debug('Exception raised when running callback function: %s' % ex)
        return res

    def list_container_objects(self, *args, **kwargs):
        raise NotImplementedError('List is not implemented for http protocol')

    def delete_object(self, obj, *args, **kwargs):
        assert isinstance(obj, self._HttpSessionHandle)
        container = self._containers[obj.container_name]
        res = container.session.delete(obj.url, headers=container.get_headers(obj.url))
        if res.status_code != requests.codes.ok:
            self._get_logger().warning('Failed deleting object %s (%d): %s' % (
                obj.object_name, res.status_code, res.text))
            return False
        return True

    def get_object(self, container_name, object_name, *args, **kwargs):
        is_stream = kwargs.get('stream', True)
        url = ''.join((container_name, object_name.lstrip('/')))
        return self._HttpSessionHandle(url, is_stream, container_name, object_name)

    def _get_download_object(self, obj):
        # bypass for session result
        if not isinstance(obj, self._HttpSessionHandle):
            return obj

        container = self._containers[obj.container_name]
        # set stream flag before we send the request
        container.session.stream = obj.is_stream
        res = container.session.get(
            obj.url, timeout=(int(self.timeout_connection), int(self.timeout_total)),
            headers=container.get_headers(obj.url))
        if res.status_code != requests.codes.ok:
            raise ValueError('Failed getting object %s (%d): %s' % (obj.object_name, res.status_code, res.text))
        return res

    def download_object_as_stream(self, obj, chunk_size=64 * 1024, **_):
        # return iterable object
        obj = self._get_download_object(obj)
        return obj.iter_content(chunk_size=chunk_size)

    def download_object(self, obj, local_path, overwrite_existing=True, delete_on_failure=True, callback=None, **_):
        obj = self._get_download_object(obj)
        p = Path(local_path)
        if not overwrite_existing and p.is_file():
            self.get_logger().warning('failed saving after download: overwrite=False and file exists (%s)' % str(p))
            return
        length = 0
        with p.open(mode='wb') as f:
            for chunk in obj.iter_content(chunk_size=5 * 1024 * 1024):
                # filter out keep-alive new chunks
                if not chunk:
                    continue
                chunk_size = len(chunk)
                f.write(chunk)
                length += chunk_size
                if callback:
                    callback(chunk_size)

        return length

    def get_direct_access(self, remote_path, **_):
        return None

    def test_upload(self, test_path, config, **kwargs):
        return True

    def upload_object(self, file_path, container, object_name, extra, callback=None, **kwargs):
        with open(file_path, 'rb') as stream:
            return self.upload_object_via_stream(iterator=stream, container=container,
                                                 object_name=object_name, extra=extra,  callback=callback, **kwargs)


class _Stream(object):
    encoding = None
    mode = 'rw'
    name = ''
    newlines = '\n'
    softspace = False

    def __init__(self, input_iterator=None):
        self.closed = False
        self._buffer = Queue()
        self._input_iterator = input_iterator
        self._leftover = None

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def close(self):
        self.closed = True

    def flush(self):
        pass

    def fileno(self):
        return 87

    def isatty(self):
        return False

    def next(self):
        while not self.closed or not self._buffer.empty():
            # input stream
            if self._input_iterator:
                try:
                    chunck = next(self._input_iterator)
                    return chunck
                except StopIteration:
                    self.closed = True
                    raise StopIteration()
                except Exception as ex:
                    _Driver.get_logger().error('Failed downloading: %s' % ex)
            else:
                # in/out stream
                try:
                    return self._buffer.get(block=True, timeout=1.)
                except Empty:
                    pass

        raise StopIteration()

    def read(self, size=None):
        try:
            data = self.next() if self._leftover is None else self._leftover
        except StopIteration:
            return six.b('')

        self._leftover = None
        try:
            while size is None or not data or len(data) < size:
                chunk = self.next()
                if chunk is not None:
                    if data is not None:
                        data += chunk
                    else:
                        data = chunk
        except StopIteration:
            pass

        if size is not None and data and len(data) > size:
            self._leftover = data[size:]
            return data[:size]

        return data

    def readline(self, size=None):
        return self.read(size)

    def readlines(self, sizehint=None):
        pass

    def truncate(self, size=None):
        pass

    def write(self, bytes):
        self._buffer.put(bytes, block=True)

    def writelines(self, sequence):
        for s in sequence:
            self.write(s)


class _Boto3Driver(_Driver):
    """ Boto3 storage adapter (simple, enough for now) """

    _min_pool_connections = 512
    _max_multipart_concurrency = deferred_config('aws.boto3.max_multipart_concurrency', 16)
    _pool_connections = deferred_config('aws.boto3.pool_connections', 512)
    _connect_timeout = deferred_config('aws.boto3.connect_timeout', 60)
    _read_timeout = deferred_config('aws.boto3.read_timeout', 60)

    _stream_download_pool_connections = deferred_config('aws.boto3.stream_connections', 128)
    _stream_download_pool = None
    _stream_download_pool_pid = None

    _containers = {}

    scheme = 's3'
    scheme_prefix = str(furl(scheme=scheme, netloc=''))

    _bucket_location_failure_reported = set()

    class _Container(object):
        _creation_lock = ForkSafeRLock()

        def __init__(self, name, cfg):
            try:
                import boto3
                import botocore.client
                from botocore.exceptions import ClientError  # noqa: F401
            except ImportError:
                raise UsageError(
                    'AWS S3 storage driver (boto3) not found. '
                    'Please install driver using: pip install \"boto3>=1.9\"'
                )

            # skip 's3://'
            self.name = name[5:]
            endpoint = (('https://' if cfg.secure else 'http://') + cfg.host) if cfg.host else None

            # boto3 client creation isn't thread-safe (client itself is)
            with self._creation_lock:
                boto_kwargs = {
                    "endpoint_url": endpoint,
                    "use_ssl": cfg.secure,
                    "verify": cfg.verify,
                    "config": botocore.client.Config(
                        max_pool_connections=max(
                            int(_Boto3Driver._min_pool_connections),
                            int(_Boto3Driver._pool_connections)),
                        connect_timeout=int(_Boto3Driver._connect_timeout),
                        read_timeout=int(_Boto3Driver._read_timeout),
                    )
                }
                if not cfg.use_credentials_chain:
                    boto_kwargs["aws_access_key_id"] = cfg.key
                    boto_kwargs["aws_secret_access_key"] = cfg.secret

                self.resource = boto3.resource(
                    's3',
                    **boto_kwargs
                )

                self.config = cfg
                bucket_name = self.name[len(cfg.host) + 1:] if cfg.host else self.name
                self.bucket = self.resource.Bucket(bucket_name)

    @attrs
    class ListResult(object):
        name = attrib(default=None)

    def __init__(self):
        pass

    def _get_stream_download_pool(self):
        if self._stream_download_pool is None or self._stream_download_pool_pid != os.getpid():
            self._stream_download_pool_pid = os.getpid()
            self._stream_download_pool = ThreadPoolExecutor(max_workers=int(self._stream_download_pool_connections))
        return self._stream_download_pool

    def get_container(self, container_name, config=None, **kwargs):
        if container_name not in self._containers:
            self._containers[container_name] = self._Container(name=container_name, cfg=config)
        self._containers[container_name].config.retries = kwargs.get('retries', 5)
        return self._containers[container_name]

    def upload_object_via_stream(self, iterator, container, object_name, callback=None, extra=None, **kwargs):
        import boto3.s3.transfer
        stream = _Stream(iterator)
        try:
            container.bucket.upload_fileobj(stream, object_name, Config=boto3.s3.transfer.TransferConfig(
                use_threads=container.config.multipart,
                max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
                num_download_attempts=container.config.retries),
                Callback=callback,
            )
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
            return False
        return True

    def upload_object(self, file_path, container, object_name, callback=None, extra=None, **kwargs):
        import boto3.s3.transfer
        try:
            container.bucket.upload_file(file_path, object_name, Config=boto3.s3.transfer.TransferConfig(
                use_threads=container.config.multipart,
                max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
                num_download_attempts=container.config.retries),
                Callback=callback)
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
            return False
        return True

    def list_container_objects(self, container, ex_prefix=None, **kwargs):
        if ex_prefix:
            res = container.bucket.objects.filter(Prefix=ex_prefix)
        else:
            res = container.bucket.objects.all()
        for res in res:
            yield self.ListResult(name=res.key)

    def delete_object(self, object, **kwargs):
        from botocore.exceptions import ClientError
        object.delete()
        try:
            # Try loading the file to verify deletion
            object.load()
            return False
        except ClientError as e:
            return int(e.response['Error']['Code']) == 404

    def get_object(self, container_name, object_name, *args, **kwargs):
        full_container_name = 's3://' + container_name
        container = self._containers[full_container_name]
        obj = container.resource.Object(container.bucket.name, object_name)
        obj.container_name = full_container_name
        return obj

    def download_object_as_stream(self, obj, chunk_size=64 * 1024, verbose=None, log=None, **_):
        def async_download(a_obj, a_stream, cb, cfg):
            try:
                a_obj.download_fileobj(a_stream, Callback=cb, Config=cfg)
            except Exception as ex:
                (log or self.get_logger()).error('Failed downloading: %s' % ex)
            a_stream.close()

        import boto3.s3.transfer
        # return iterable object
        stream = _Stream()
        container = self._containers[obj.container_name]
        config = boto3.s3.transfer.TransferConfig(
            use_threads=container.config.multipart,
            max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
            num_download_attempts=container.config.retries)
        total_size_mb = obj.content_length / (1024. * 1024.)
        remote_path = os.path.join(obj.container_name, obj.key)
        cb = DownloadProgressReport(total_size_mb, verbose, remote_path, log)
        self._get_stream_download_pool().submit(async_download, obj, stream, cb, config)

        return stream

    def download_object(self, obj, local_path, overwrite_existing=True, delete_on_failure=True, callback=None, **_):
        import boto3.s3.transfer
        p = Path(local_path)
        if not overwrite_existing and p.is_file():
            self.get_logger().warning('failed saving after download: overwrite=False and file exists (%s)' % str(p))
            return
        container = self._containers[obj.container_name]
        Config = boto3.s3.transfer.TransferConfig(
            use_threads=container.config.multipart,
            max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
            num_download_attempts=container.config.retries
        )
        obj.download_file(str(p), Callback=callback, Config=Config)

    @classmethod
    def _test_bucket_config(cls, conf, log, test_path='', raise_on_error=True, log_on_error=True):
        try:
            import boto3
            from botocore.exceptions import ClientError
        except ImportError:
            return False

        if not conf.bucket:
            return False
        try:
            if not conf.is_valid():
                raise Exception('Missing credentials')

            fullname = furl(conf.bucket).add(path=test_path).add(path='%s-upload_test' % cls.__module__)
            bucket_name = str(fullname.path.segments[0])
            filename = str(furl(path=fullname.path.segments[1:]))

            data = {
                'user': getpass.getuser(),
                'machine': gethostname(),
                'time': datetime.utcnow().isoformat()
            }

            boto_session = boto3.Session(conf.key, conf.secret)
            boto_resource = boto_session.resource('s3', conf.region)
            bucket = boto_resource.Bucket(bucket_name)
            bucket.put_object(Key=filename, Body=six.b(json.dumps(data)))

            region = cls._get_bucket_region(conf=conf, log=log, report_info=True)

            if region and ((conf.region and region != conf.region) or (not conf.region and region != 'us-east-1')):
                msg = "incorrect region specified for bucket %s (detected region %s)" % (conf.bucket, region)
            else:
                return True

        except ClientError as ex:
            msg = ex.response['Error']['Message']
            if log_on_error and log:
                log.error(msg)

            if raise_on_error:
                raise

        except Exception as ex:
            msg = str(ex)
            if log_on_error and log:
                log.error(msg)

            if raise_on_error:
                raise

        msg = ("Failed testing access to bucket %s: " % conf.bucket) + msg

        if log_on_error and log:
            log.error(msg)

        if raise_on_error:
            raise StorageError(msg)

        return False

    @classmethod
    def _get_bucket_region(cls, conf, log=None, report_info=False):
        import boto3
        from botocore.exceptions import ClientError

        if not conf.bucket:
            return None

        def report(msg):
            if log and conf.get_bucket_host() not in cls._bucket_location_failure_reported:
                if report_info:
                    log.debug(msg)
                else:
                    log.warning(msg)
                cls._bucket_location_failure_reported.add(conf.get_bucket_host())

        try:
            boto_session = boto3.Session(conf.key, conf.secret)
            boto_resource = boto_session.resource('s3')
            return boto_resource.meta.client.get_bucket_location(Bucket=conf.bucket)["LocationConstraint"]

        except ClientError as ex:
            report("Failed getting bucket location (region) for bucket "
                   "%s: %s (%s, access_key=%s). Default region will be used. "
                   "This is normal if you do not have GET_BUCKET_LOCATION permission"
                   % (conf.bucket, ex.response['Error']['Message'], ex.response['Error']['Code'], conf.key))
        except Exception as ex:
            report("Failed getting bucket location (region) for bucket %s: %s. Default region will be used."
                   % (conf.bucket, str(ex)))

        return None

    def get_direct_access(self, remote_path, **_):
        return None

    def test_upload(self, test_path, config, **_):
        return True


class _GoogleCloudStorageDriver(_Driver):
    """Storage driver for google cloud storage"""

    _stream_download_pool_connections = deferred_config('google.storage.stream_connections', 128)
    _stream_download_pool = None
    _stream_download_pool_pid = None

    _containers = {}

    scheme = 'gs'
    scheme_prefix = str(furl(scheme=scheme, netloc=''))

    class _Container(object):
        def __init__(self, name, cfg):
            try:
                from google.cloud import storage
                from google.oauth2 import service_account
            except ImportError:
                raise UsageError(
                    'Google cloud driver not found. '
                    'Please install driver using: pip install \"google-cloud-storage>=1.13.2\"'
                )

            self.name = name[len(_GoogleCloudStorageDriver.scheme_prefix):]

            if cfg.credentials_json:
                credentials = service_account.Credentials.from_service_account_file(cfg.credentials_json)
            else:
                credentials = None

            self.client = storage.Client(project=cfg.project, credentials=credentials)
            for adapter in self.client._http.adapters.values():
                if cfg.pool_connections:
                    adapter._pool_connections = cfg.pool_connections
                if cfg.pool_maxsize:
                    adapter._pool_maxsize = cfg.pool_maxsize

            self.config = cfg
            self.bucket = self.client.bucket(self.name)

    def _get_stream_download_pool(self):
        if self._stream_download_pool is None or self._stream_download_pool_pid != os.getpid():
            self._stream_download_pool_pid = os.getpid()
            self._stream_download_pool = ThreadPoolExecutor(max_workers=int(self._stream_download_pool_connections))
        return self._stream_download_pool

    def get_container(self, container_name, config=None, **kwargs):
        if container_name not in self._containers:
            self._containers[container_name] = self._Container(name=container_name, cfg=config)
        self._containers[container_name].config.retries = kwargs.get('retries', 5)
        return self._containers[container_name]

    def upload_object_via_stream(self, iterator, container, object_name, extra=None, **kwargs):
        try:
            blob = container.bucket.blob(object_name)
            blob.upload_from_file(iterator)
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
            return False
        return True

    def upload_object(self, file_path, container, object_name, extra=None, **kwargs):
        try:
            blob = container.bucket.blob(object_name)
            blob.upload_from_filename(file_path)
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
            return False
        return True

    def list_container_objects(self, container, **kwargs):
        return list(container.bucket.list_blobs())

    def delete_object(self, object, **kwargs):
        try:
            object.delete()
        except Exception as ex:
            try:
                from google.cloud.exceptions import NotFound
                if isinstance(ex, NotFound):
                    return False
            except ImportError:
                pass
            name = getattr(object, "name", "")
            self.get_logger().warning("Failed deleting object {}: {}".format(name, ex))
            return False

        return not object.exists()

    def get_object(self, container_name, object_name, *args, **kwargs):
        full_container_name = str(furl(scheme=self.scheme, netloc=container_name))
        container = self._containers[full_container_name]
        obj = container.bucket.blob(object_name)
        obj.container_name = full_container_name
        return obj

    def download_object_as_stream(self, obj, chunk_size=256 * 1024, **_):
        raise NotImplementedError('Unsupported for google storage')

        def async_download(a_obj, a_stream):
            try:
                a_obj.download_to_file(a_stream)
            except Exception as ex:
                self.get_logger().error('Failed downloading: %s' % ex)
            a_stream.close()

        # return iterable object
        stream = _Stream()
        obj.chunk_size = chunk_size
        self._get_stream_download_pool().submit(async_download, obj, stream)

        return stream

    def download_object(self, obj, local_path, overwrite_existing=True, delete_on_failure=True, callback=None, **_):
        p = Path(local_path)
        if not overwrite_existing and p.is_file():
            self.get_logger().warning('failed saving after download: overwrite=False and file exists (%s)' % str(p))
            return
        obj.download_to_filename(str(p))

    def test_upload(self, test_path, config, **_):
        bucket_url = str(furl(scheme=self.scheme, netloc=config.bucket, path=config.subdir))
        bucket = self.get_container(container_name=bucket_url, config=config).bucket

        test_obj = bucket

        if test_path:
            if not test_path.endswith('/'):
                test_path += '/'

            blob = bucket.blob(test_path)

            if blob.exists():
                test_obj = blob

        permissions_to_test = ('storage.objects.get', 'storage.objects.update')
        return set(test_obj.test_iam_permissions(permissions_to_test)) == set(permissions_to_test)

    def get_direct_access(self, remote_path, **_):
        return None


class _AzureBlobServiceStorageDriver(_Driver):
    scheme = 'azure'

    _containers = {}

    class _Container(object):
        def __init__(self, name, config):
            try:
                from azure.common import AzureHttpError  # noqa: F401
                from azure.storage.blob import BlockBlobService
            except ImportError:
                raise UsageError(
                    'Azure blob storage driver not found. '
                    'Please install driver using: pip install \"azure.storage.blob<=2.1.0\"'
                )

            self.name = name
            self.config = config
            self.blob_service = BlockBlobService(
                account_name=config.account_name,
                account_key=config.account_key,
            )

    @attrs
    class _Object(object):
        container = attrib()
        blob_name = attrib()
        content_length = attrib()

    def get_container(self, container_name=None, config=None, **kwargs):
        container_name = container_name or config.container_name
        if container_name not in self._containers:
            self._containers[container_name] = self._Container(name=container_name, config=config)
        # self._containers[container_name].config.retries = kwargs.get('retries', 5)
        return self._containers[container_name]

    def upload_object_via_stream(self, iterator, container, object_name, callback=None, extra=None, **kwargs):
        from azure.common import AzureHttpError  # noqa

        blob_name = self._blob_name_from_object_path(object_name, container.name)  # noqa: F841
        try:
            container.blob_service.MAX_SINGLE_PUT_SIZE = 16 * 1024 * 1024
            container.blob_service.socket_timeout = (300, 2000)
            container.blob_service.create_blob_from_bytes(
                container.name,
                object_name,
                iterator.read() if hasattr(iterator, "read") else bytes(iterator),
                # timeout=300,
                max_connections=2,
                progress_callback=callback,
            )
            return True
        except AzureHttpError as ex:
            self.get_logger().error('Failed uploading (Azure error): %s' % ex)
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
        return False

    def upload_object(self, file_path, container, object_name, callback=None, extra=None, **kwargs):
        from azure.common import AzureHttpError  # noqa

        blob_name = self._blob_name_from_object_path(object_name, container.name)
        stream = None
        try:
            from azure.storage.blob import ContentSettings  # noqa
            from mimetypes import guess_type
            container.blob_service.MAX_SINGLE_PUT_SIZE = 16 * 1024 * 1024
            container.blob_service.socket_timeout = (300, 2000)
            container.blob_service.create_blob_from_path(
                container.name,
                blob_name,
                file_path,
                # timeout=300,
                max_connections=2,
                content_settings=ContentSettings(content_type=guess_type(file_path)),
                progress_callback=callback,
            )
            return True
        except AzureHttpError as ex:
            self.get_logger().error('Failed uploading (Azure error): %s' % ex)
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
        finally:
            if stream:
                stream.close()

    def list_container_objects(self, container, ex_prefix=None, **kwargs):
        return list(container.blob_service.list_blobs(container_name=container.name, prefix=ex_prefix))

    def delete_object(self, object, **kwargs):
        container = object.container
        container.blob_service.delete_blob(
            container.name,
            object.blob_name,
        )
        return not object.container.blob_service.exists(container.name, object.blob_name)

    def get_object(self, container_name, object_name, *args, **kwargs):
        container = self._containers.get(container_name)
        if not container:
            raise StorageError("Container `{}` not found for object {}".format(container_name, object_name))

        # blob_name = self._blob_name_from_object_path(object_name, container_name)
        blob = container.blob_service.get_blob_properties(container.name, object_name)

        return self._Object(container=container, blob_name=blob.name, content_length=blob.properties.content_length)

    def download_object_as_stream(self, obj, verbose, *_, **__):
        container = obj.container
        total_size_mb = obj.content_length / (1024. * 1024.)
        remote_path = os.path.join(
            "{}://".format(self.scheme),
            container.config.account_name,
            container.name,
            obj.blob_name
        )
        cb = DownloadProgressReport(total_size_mb, verbose, remote_path, self.get_logger())
        blob = container.blob_service.get_blob_to_bytes(
            container.name,
            obj.blob_name,
            progress_callback=cb,
        )
        return blob.content

    def download_object(self, obj, local_path, overwrite_existing=True, delete_on_failure=True, callback=None, **_):
        p = Path(local_path)
        if not overwrite_existing and p.is_file():
            self.get_logger().warning('failed saving after download: overwrite=False and file exists (%s)' % str(p))
            return

        download_done = threading.Event()
        download_done.counter = 0

        def callback_func(current, total):
            if callback:
                chunk = current - download_done.counter
                download_done.counter += chunk
                callback(chunk)
            if current >= total:
                download_done.set()

        container = obj.container
        container.blob_service.MAX_SINGLE_GET_SIZE = 5 * 1024 * 1024
        _ = container.blob_service.get_blob_to_path(
            container.name,
            obj.blob_name,
            local_path,
            max_connections=10,
            progress_callback=callback_func,
        )
        download_done.wait()

    def test_upload(self, test_path, config, **_):
        container = self.get_container(config=config)
        try:
            container.blob_service.get_container_properties(container.name)
        except Exception:
            return False
        else:
            # Using the account Key, we can always upload...
            return True

    @classmethod
    def _blob_name_from_object_path(cls, name, container_name):
        scheme = urlparse(name).scheme
        if scheme:
            if scheme != cls.scheme:
                raise StorageError(
                    "When using a URL, only the `{}` scheme is supported for Azure storage: {}",
                    cls.scheme,
                    name,
                )

            f = furl(name)

            if not f.path.segments:
                raise StorageError(
                    "Missing container name in URL {}",
                    name,
                )

            parsed_container_name = f.path.segments[0]

            if parsed_container_name != container_name:
                raise StorageError(
                    "Container name mismatch (expected {}, found {}) in {}",
                    container_name,
                    parsed_container_name,
                    name,
                )

            if len(f.path.segments) == 1:
                raise StorageError(
                    "No path found following container name {} in {}",
                    container_name,
                    name,
                )

            return f.path.segments[0], os.path.join(*f.path.segments[1:])

        return name

    def get_direct_access(self, remote_path, **_):
        return None


class _FileStorageDriver(_Driver):
    """
    A base StorageDriver to derive from.
    """

    scheme = "file"
    CHUNK_SIZE = 8096
    IGNORE_FOLDERS = ['.lock', '.hash']
    Object = namedtuple("Object", ['name', 'size', 'extra', 'driver', 'container', 'hash', 'meta_data'])

    class _Container(object):
        def __init__(self, name, extra, driver):
            self.name = name
            self.extra = extra
            self.driver = driver

    def __init__(self, key, secret=None, secure=True, host=None, port=None,
                 **kwargs):

        # Use the key as the path to the storage
        self.base_path = key

    def _make_path(self, path, ignore_existing=True):
        """
        Create a path by checking if it already exists
        """

        try:
            os.makedirs(path)
        except OSError:
            exp = sys.exc_info()[1]
            if exp.errno == errno.EEXIST and not ignore_existing:
                raise exp

    def _check_container_name(self, container_name):
        """
        Check if the container name is valid

        :param container_name: Container name
        :type container_name: ``str``
        """

        if '/' in container_name or '\\' in container_name:
            raise ValueError("Container name \"{}\" cannot contain \\ or / ".format(container_name))

    def _make_container(self, container_name):
        """
        Create a container instance

        :param container_name: Container name.
        :type container_name: ``str``

        :return: A Container instance.
        """
        container_name = container_name or '.'
        self._check_container_name(container_name)

        full_path = os.path.realpath(os.path.join(self.base_path, container_name))

        try:
            stat = os.stat(full_path)
            if not os.path.isdir(full_path):
                raise OSError("Target path \"{}\" is not a directory".format(full_path))
        except OSError:
            raise OSError("Target path \"{}\" is not accessible or does not exist".format(full_path))

        extra = {
            'creation_time': stat.st_ctime,
            'access_time': stat.st_atime,
            'modify_time': stat.st_mtime,
        }

        return self._Container(name=container_name, extra=extra, driver=self)

    def _make_object(self, container, object_name):
        """
        Create an object instance

        :param container: Container.
        :type container: :class:`Container`

        :param object_name: Object name.
        :type object_name: ``str``

        :return: A Object instance.
        """

        full_path = os.path.realpath(os.path.join(self.base_path, container.name if container else '.', object_name))

        if os.path.isdir(full_path):
            raise ValueError("Target path \"{}\" already exist".format(full_path))

        try:
            stat = os.stat(full_path)
        except Exception:
            raise ValueError("Cannot access target path \"{}\"".format(full_path))

        extra = {
            'creation_time': stat.st_ctime,
            'access_time': stat.st_atime,
            'modify_time': stat.st_mtime,
        }

        return self.Object(name=object_name, size=stat.st_size, extra=extra,
                           driver=self, container=container, hash=None, meta_data=None)

    def iterate_containers(self):
        """
        Return a generator of containers.

        :return: A generator of Container instances.
        """

        for container_name in os.listdir(self.base_path):
            full_path = os.path.join(self.base_path, container_name)
            if not os.path.isdir(full_path):
                continue
            yield self._make_container(container_name)

    def _get_objects(self, container):
        """
        Recursively iterate through the file-system and return the object names
        """

        cpath = self.get_container_cdn_url(container, check=True)

        for folder, subfolders, files in os.walk(cpath, topdown=True):
            # Remove unwanted subfolders
            for subf in self.IGNORE_FOLDERS:
                if subf in subfolders:
                    subfolders.remove(subf)

            for name in files:
                full_path = os.path.join(folder, name)
                object_name = os.path.relpath(full_path, start=cpath)
                yield self._make_object(container, object_name)

    def iterate_container_objects(self, container):
        """
        Returns a generator of objects for the given container.

        :param container: Container instance
        :type container: :class:`Container`

        :return: A generator of Object instances.
        """

        return self._get_objects(container)

    def get_container(self, container_name, **_):
        """
        Return a container instance.

        :param container_name: Container name.
        :type container_name: ``str``

        :return: A Container instance.
        """
        return self._make_container(container_name)

    def get_container_cdn_url(self, container, check=False):
        """
        Return a container CDN URL.

        :param container: Container instance
        :type  container: :class:`Container`

        :param check: Indicates if the path's existence must be checked
        :type check: ``bool``

        :return: A CDN URL for this container.
        """
        path = os.path.realpath(os.path.join(self.base_path, container.name if container else '.'))

        if check and not os.path.isdir(path):
            raise ValueError("Target path \"{}\" does not exist".format(path))

        return path

    def get_object(self, container_name, object_name, **_):
        """
        Return an object instance.

        :param container_name: Container name.
        :type  container_name: ``str``

        :param object_name: Object name.
        :type  object_name: ``str``

        :return: An Object instance.
        """
        container = self._make_container(container_name)
        return self._make_object(container, object_name)

    def get_object_cdn_url(self, obj):
        """
        Return an object CDN URL.

        :param obj: Object instance
        :type  obj: :class:`Object`

        :return: A CDN URL for this object.
        """
        return os.path.realpath(os.path.join(self.base_path, obj.container.name, obj.name))

    def download_object(self, obj, destination_path, overwrite_existing=False, delete_on_failure=True, **_):
        """
        Download an object to the specified destination path.

        :param obj: Object instance.
        :type obj: :class:`Object`

        :param destination_path: Full path to a file or a directory where the
                                incoming file will be saved.
        :type destination_path: ``str``

        :param overwrite_existing: True to overwrite an existing file,
            defaults to False.
        :type overwrite_existing: ``bool``

        :param delete_on_failure: True to delete a partially downloaded file if
        the download was not successful (hash mismatch / file size).
        :type delete_on_failure: ``bool``

        :return: True, if an object has been successfully downloaded, False, otherwise.
        """

        obj_path = self.get_object_cdn_url(obj)
        base_name = os.path.basename(destination_path)

        if not base_name and not os.path.exists(destination_path):
            raise ValueError('Path \"{}\" does not exist'.format(destination_path))

        if not base_name:
            file_path = os.path.join(destination_path, obj.name)
        else:
            file_path = destination_path

        if os.path.exists(file_path) and not overwrite_existing:
            raise ValueError('File \"{}\" already exists, but overwrite_existing=False'.format(file_path))

        try:
            shutil.copy(obj_path, file_path)
        except IOError:
            if delete_on_failure:
                # noinspection PyBroadException
                try:
                    os.unlink(file_path)
                except Exception:
                    pass
            return False

        return True

    def download_object_as_stream(self, obj, chunk_size=None, **_):
        """
        Return a generator which yields object data.

        :param obj: Object instance
        :type obj: :class:`Object`

        :param chunk_size: Optional chunk size (in bytes).
        :type chunk_size: ``int``

        :return: A stream of binary chunks of data.
        """
        path = self.get_object_cdn_url(obj)
        with open(path, 'rb') as obj_file:
            for data in self._read_in_chunks(obj_file, chunk_size=chunk_size):
                yield data

    def upload_object(self, file_path, container, object_name, extra=None, verify_hash=True, **_):
        """
        Upload an object currently located on a disk.

        :param file_path: Path to the object on disk.
        :type file_path: ``str``

        :param container: Destination container.
        :type container: :class:`Container`

        :param object_name: Object name.
        :type object_name: ``str``

        :param verify_hash: Verify hast
        :type verify_hash: ``bool``

        :param extra: (optional) Extra attributes (driver specific).
        :type extra: ``dict``
        """

        path = self.get_container_cdn_url(container, check=True)
        obj_path = os.path.join(path, object_name)
        base_path = os.path.dirname(obj_path)

        self._make_path(base_path)

        shutil.copy(file_path, obj_path)

        os.chmod(obj_path, int('664', 8))

        return self._make_object(container, object_name)

    def upload_object_via_stream(self, iterator, container, object_name, extra=None, **kwargs):
        """
        Upload an object using an iterator.

        If a provider supports it, chunked transfer encoding is used and you
        don't need to know in advance the amount of data to be uploaded.

        Otherwise if a provider doesn't support it, iterator will be exhausted
        so a total size for data to be uploaded can be determined.

        Note: Exhausting the iterator means that the whole data must be
        buffered in memory which might result in memory exhausting when
        uploading a very large object.

        If a file is located on a disk you are advised to use upload_object
        function which uses fs.stat function to determine the file size and it
        doesn't need to buffer whole object in the memory.

        :type iterator: ``object``
        :param iterator: An object which implements the iterator
                         interface and yields binary chunks of data.

        :type container: :class:`Container`
        :param container: Destination container.

        :type object_name: ``str``
        :param object_name: Object name.

        :type extra: ``dict``
        :param extra: (optional) Extra attributes (driver specific). Note:
            This dictionary must contain a 'content_type' key which represents
            a content type of the stored object.
        """
        path = self.get_container_cdn_url(container, check=True)
        obj_path = os.path.join(path, object_name)
        base_path = os.path.dirname(obj_path)
        self._make_path(base_path)

        obj_path = os.path.realpath(obj_path)
        with open(obj_path, 'wb' if not isinstance(iterator, StringIO) else 'wt') as obj_file:
            obj_file.write(iterator.read() if hasattr(iterator, 'read') else bytes(iterator))

        os.chmod(obj_path, int('664', 8))
        return self._make_object(container, object_name)

    def delete_object(self, obj, **_):
        """
        Delete an object.

        :type obj: :class:`Object`
        :param obj: Object instance.

        :return: True on success.
        """
        if not obj:
            return False

        path = self.get_object_cdn_url(obj)

        try:
            os.unlink(path)
        except Exception:
            return False

        # # Check and delete all the empty parent folders
        # path = os.path.dirname(path)
        # container_url = obj.container.get_cdn_url()
        #
        # # Delete the empty parent folders till the container's level
        # while path != container_url:
        #     try:
        #         os.rmdir(path)
        #     except OSError:
        #         exp = sys.exc_info()[1]
        #         if exp.errno == errno.ENOTEMPTY:
        #             break
        #         raise exp
        #
        #     path = os.path.dirname(path)

        return True

    def create_container(self, container_name):
        """
        Create a new container.

        :type container_name: ``str``
        :param container_name: Container name.

        :return: A Container instance on success.
        """
        container_name = container_name or '.'
        self._check_container_name(container_name)

        path = os.path.join(self.base_path, container_name)

        try:
            self._make_path(path, ignore_existing=False)
        except OSError:
            exp = sys.exc_info()[1]
            if exp.errno == errno.EEXIST:
                raise ValueError('Container \"{}\" with this name already exists. The name '
                                 'must be unique among all the containers in the '
                                 'system'.format(container_name))
            else:
                raise ValueError('Error creating container \"{}\"'.format(container_name))
        except Exception:
            raise ValueError('Error creating container \"{}\"'.format(container_name))

        return self._make_container(container_name)

    def delete_container(self, container):
        """
        Delete a container.

        :type container: :class:`Container`
        :param container: Container instance

        :return: True on success, False otherwise.
        """

        # Check if there are any objects inside this
        for obj in self._get_objects(container):
            raise ValueError('Container \"{}\" is not empty'.format(container.name))

        path = self.get_container_cdn_url(container, check=True)

        # noinspection PyBroadException
        try:
            shutil.rmtree(path)
        except Exception:
            return False

        return True

    def list_container_objects(self, container, **kwargs):
        return list(self.iterate_container_objects(container))

    @staticmethod
    def _read_in_chunks(iterator, chunk_size=None, fill_size=False, yield_empty=False):
        """
        Return a generator which yields data in chunks.

        :param iterator: An object which implements an iterator interface
                         or a File like object with read method.
        :type iterator: :class:`object` which implements iterator interface.

        :param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
        :type chunk_size: ``int``

        :param fill_size: If True, make sure chunks are exactly chunk_size in
                          length (except for last chunk).
        :type fill_size: ``bool``

        :param yield_empty: If true and iterator returned no data, only yield empty
                            bytes object
        :type yield_empty: ``bool``

        TODO: At some point in the future we could use byte arrays here if version
        >= Python 3. This should speed things up a bit and reduce memory usage.
        """
        chunk_size = chunk_size or _FileStorageDriver.CHUNK_SIZE
        if six.PY3:
            from io import FileIO as file

        if isinstance(iterator, (file)):
            get_data = iterator.read
            args = (chunk_size,)
        else:
            get_data = next
            args = (iterator,)

        data = bytes('')
        empty = False

        while not empty or len(data) > 0:
            if not empty:
                try:
                    chunk = bytes(get_data(*args))
                    if len(chunk) > 0:
                        data += chunk
                    else:
                        empty = True
                except StopIteration:
                    empty = True

            if len(data) == 0:
                if empty and yield_empty:
                    yield bytes('')

                return

            if fill_size:
                if empty or len(data) >= chunk_size:
                    yield data[:chunk_size]
                    data = data[chunk_size:]
            else:
                yield data
                data = bytes('')

    def get_direct_access(self, remote_path, **_):
        # this will always make sure we have full path and file:// prefix
        full_url = StorageHelper.conform_url(remote_path)
        # now get rid of the file:// prefix
        path = Path(full_url[7:])
        if not path.exists():
            raise ValueError("Requested path does not exist: {}".format(path))
        return path.as_posix()

    def test_upload(self, test_path, config, **kwargs):
        return True


driver_schemes = set(
    filter(
        None,
        itertools.chain(
            (getattr(cls, "scheme", None) for cls in _Driver.__subclasses__()),
            *(getattr(cls, "schemes", []) for cls in _Driver.__subclasses__())
        )
    )
)

remote_driver_schemes = driver_schemes - {_FileStorageDriver.scheme}


# clearml/storage/manager.py

import fnmatch
import os
import shutil
import tarfile
from multiprocessing.pool import ThreadPool
from random import random
from time import time
from typing import List, Optional
from zipfile import ZipFile

from pathlib2 import Path

from .cache import CacheManager
from .helper import StorageHelper
from .util import encode_string_to_filename
from ..debugging.log import LoggerRoot


class StorageManager(object):
    """
    StorageManager is helper interface for downloading & uploading files to supported remote storage
    Support remote servers: http(s)/S3/GS/Azure/File-System-Folder
    Cache is enabled by default for all downloaded remote urls/files
    """

    @classmethod
    def get_local_copy(
        cls, remote_url, cache_context=None, extract_archive=True, name=None, force_download=False,
    ):
        # type: (str, Optional[str], bool, Optional[str], bool) -> str
        """
        Get a local copy of the remote file. If the remote URL is a direct file access,
        the returned link is the same, otherwise a link to a local copy of the url file is returned.
        Caching is enabled by default, cache limited by number of stored files per cache context.
        Oldest accessed files are deleted when cache is full.

        :param str remote_url: remote url link (string)
        :param str cache_context: Optional caching context identifier (string), default context 'global'
        :param bool extract_archive: if True returned path will be a cached folder containing the archive's content,
            currently only zip files are supported.
        :param str name: name of the target file
        :param bool force_download: download file from remote even if exists in local cache
        :return: Full path to local copy of the requested url. Return None on Error.
        """
        cache = CacheManager.get_cache_manager(cache_context=cache_context)
        cached_file = cache.get_local_copy(remote_url=remote_url, force_download=force_download)
        if extract_archive and cached_file:
            # this will get us the actual cache (even with direct access)
            cache_path_encoding = Path(cache.get_cache_folder()) / cache.get_hashed_url_file(remote_url)
            return cls._extract_to_cache(
                cached_file, name, cache_context, cache_path_encoding=cache_path_encoding.as_posix())

        return cached_file

    @classmethod
    def upload_file(
        cls, local_file, remote_url, wait_for_upload=True, retries=1
    ):  # type: (str, str, bool, int) -> str
        """
        Upload a local file to a remote location. remote url is the finale destination of the uploaded file.

        Examples:

        .. code-block:: py

            upload_file('/tmp/artifact.yaml', 'http://localhost:8081/manual_artifacts/my_artifact.yaml')
            upload_file('/tmp/artifact.yaml', 's3://a_bucket/artifacts/my_artifact.yaml')
            upload_file('/tmp/artifact.yaml', '/mnt/share/folder/artifacts/my_artifact.yaml')

        :param str local_file: Full path of a local file to be uploaded
        :param str remote_url: Full path or remote url to upload to (including file name)
        :param bool wait_for_upload: If False, return immediately and upload in the background. Default True.
        :param int retries: Number of retries before failing to upload file, default 1.
        :return: Newly uploaded remote URL.
        """
        return CacheManager.get_cache_manager().upload_file(
            local_file=local_file,
            remote_url=remote_url,
            wait_for_upload=wait_for_upload,
            retries=retries,
        )

    @classmethod
    def set_cache_file_limit(
        cls, cache_file_limit, cache_context=None
    ):  # type: (int, Optional[str]) -> int
        """
        Set the cache context file limit. File limit is the maximum number of files the specific cache context holds.
        Notice, there is no limit on the size of these files, only the total number of cached files.

        :param int cache_file_limit: New maximum number of cached files
        :param str cache_context: Optional cache context identifier, default global context
        :return: The new cache context file limit.
        """
        return CacheManager.get_cache_manager(
            cache_context=cache_context, cache_file_limit=cache_file_limit
        ).set_cache_limit(cache_file_limit)

    @classmethod
    def _extract_to_cache(
            cls,
            cached_file,  # type: str
            name,  # type: str
            cache_context=None,  # type: Optional[str]
            target_folder=None,  # type: Optional[str]
            cache_path_encoding=None,  # type: Optional[str]
            force=False,  # type: bool
    ):
        # type: (...) -> str
        """
        Extract cached file to cache folder
        :param str cached_file: local copy of archive file
        :param str name: name of the target file
        :param str cache_context: cache context id
        :param str target_folder: specify target path to use for archive extraction
        :param str cache_path_encoding: specify representation of the local path of the cached files,
            this will always point to local cache folder, even if we have direct access file.
            Used for extracting the cached archived based on cache_path_encoding
        :param bool force: Force archive extraction even if target folder exists
        :return: cached folder containing the extracted archive content
        """
        if not cached_file:
            return cached_file

        cached_file = Path(cached_file)
        cache_path_encoding = Path(cache_path_encoding) if cache_path_encoding else None

        # we support zip and tar.gz files auto-extraction
        suffix = cached_file.suffix.lower()
        if suffix == '.gz':
            suffix = ''.join(a.lower() for a in cached_file.suffixes[-2:])

        if suffix not in (".zip", ".tgz", ".tar.gz"):
            return str(cached_file)

        cache_folder = Path(cache_path_encoding or cached_file).parent
        archive_suffix = (cache_path_encoding or cached_file).name[:-len(suffix)]
        name = encode_string_to_filename(name) if name else name
        if target_folder:
            target_folder = Path(target_folder)
        else:
            target_folder = cache_folder / CacheManager.get_context_folder_lookup(
                cache_context).format(archive_suffix, name)

        if target_folder.is_dir() and not force:
            # noinspection PyBroadException
            try:
                target_folder.touch(exist_ok=True)
                return target_folder.as_posix()
            except Exception:
                pass

        base_logger = LoggerRoot.get_base_logger()
        try:
            # if target folder exists, meaning this is forced ao we extract directly into target folder
            if target_folder.is_dir():
                temp_target_folder = target_folder
            else:
                temp_target_folder = cache_folder / "{0}_{1}_{2}".format(
                    target_folder.name, time() * 1000, str(random()).replace('.', ''))
                temp_target_folder.mkdir(parents=True, exist_ok=True)

            if suffix == ".zip":
                ZipFile(cached_file.as_posix()).extractall(path=temp_target_folder.as_posix())
            elif suffix == ".tar.gz":
                with tarfile.open(cached_file.as_posix()) as file:
                    file.extractall(temp_target_folder.as_posix())
            elif suffix == ".tgz":
                with tarfile.open(cached_file.as_posix(), mode='r:gz') as file:
                    file.extractall(temp_target_folder.as_posix())

            if temp_target_folder != target_folder:
                # we assume we will have such folder if we already extract the file
                # noinspection PyBroadException
                try:
                    # if rename fails, it means that someone else already manged to extract the file, delete the current
                    # folder and return the already existing cached zip folder
                    shutil.move(temp_target_folder.as_posix(), target_folder.as_posix())
                except Exception:
                    if target_folder.exists():
                        target_folder.touch(exist_ok=True)
                    else:
                        base_logger.warning(
                            "Failed renaming {0} to {1}".format(temp_target_folder.as_posix(), target_folder.as_posix()))
                    try:
                        shutil.rmtree(temp_target_folder.as_posix())
                    except Exception as ex:
                        base_logger.warning(
                            "Exception {}\nFailed deleting folder {}".format(ex, temp_target_folder.as_posix()))
        except Exception as ex:
            # failed extracting the file:
            base_logger.warning(
                "Exception {}\nFailed extracting zip file {}".format(ex, cached_file.as_posix()))
            # noinspection PyBroadException
            try:
                target_folder.rmdir()
            except Exception:
                pass
            return cached_file.as_posix()
        return target_folder.as_posix()

    @classmethod
    def get_files_server(cls):
        from ..backend_api import Session
        return Session.get_files_server_host()

    @classmethod
    def upload_folder(cls, local_folder, remote_url, match_wildcard=None):
        # type: (str, str, Optional[str]) -> None
        """
        Upload local folder recursively to a remote storage, maintaining the sub folder structure
        in the remote storage.

        .. note::

            If we have a local file `~/folder/sub/file.ext` then
            `StorageManager.upload_folder('~/folder/', 's3://bucket/')`
            will create `s3://bucket/sub/file.ext`

        :param str local_folder: Local folder to recursively upload
        :param str remote_url: Target remote storage location, tree structure of `local_folder` will
            be created under the target remote_url. Supports Http/S3/GS/Azure and shared filesystem.
            Example: 's3://bucket/data/'
        :param str match_wildcard: If specified only upload files matching the `match_wildcard`
            Example: `*.json`
            Notice: target file size/date are not checked. Default True, always upload.
            Notice if uploading to http, we will always overwrite the target.
        """

        base_logger = LoggerRoot.get_base_logger()

        if not Path(local_folder).is_dir():
            base_logger.error("Local folder '{}' does not exist".format(local_folder))
            return
        results = []
        helper = StorageHelper.get(remote_url)
        with ThreadPool() as pool:
            for path in Path(local_folder).rglob(match_wildcard or "*"):
                if not path.is_file():
                    continue
                results.append(
                    pool.apply_async(
                        helper.upload,
                        args=(str(path), str(path).replace(local_folder, remote_url)),
                    )
                )
            for res in results:
                res.wait()

    @classmethod
    def download_folder(
        cls, remote_url, local_folder=None, match_wildcard=None, overwrite=False
    ):
        # type: (str, Optional[str], Optional[str], bool) -> Optional[str]
        """
        Download remote folder recursively to the local machine, maintaining the sub folder structure
        from the remote storage.

        .. note::

            If we have a local file `s3://bucket/sub/file.ext` then
            `StorageManager.download_folder('s3://bucket/', '~/folder/')`
            will create `~/folder/sub/file.ext`

        :param str remote_url: Source remote storage location, tree structure of `remote_url` will
            be created under the target local_folder. Supports S3/GS/Azure and shared filesystem.
            Example: 's3://bucket/data/'
        :param str local_folder: Local target folder to create the full tree from remote_url.
            If None, use the cache folder. (Default: use cache folder)
        :param match_wildcard: If specified only download files matching the `match_wildcard`
            Example: `*.json`
        :param bool overwrite: If False, and target files exist do not download.
            If True always download the remote files. Default False.

        :return: Target local folder
        """

        base_logger = LoggerRoot.get_base_logger()

        if local_folder:
            try:
                Path(local_folder).mkdir(parents=True, exist_ok=True)
            except OSError as ex:
                base_logger.error("Failed creating local folder '{}': {}".format(local_folder, ex))
                return
        else:
            local_folder = CacheManager.get_cache_manager().get_cache_folder()

        helper = StorageHelper.get(remote_url)
        results = []

        with ThreadPool() as pool:
            for path in helper.list(prefix=remote_url):
                remote_path = str(Path(helper.base_url) / Path(path)) \
                    if helper.get_driver_direct_access(helper.base_url) else \
                    "{}/{}".format(helper.base_url.rstrip('/'), path.lstrip('/'))
                if match_wildcard and not fnmatch.fnmatch(remote_path, match_wildcard):
                    continue
                local_url = os.path.join(
                    str(Path(local_folder)),
                    str(Path(remote_path[len(remote_url):].lstrip(os.path.sep)))
                )
                if not os.path.exists(local_url) or os.path.getsize(local_url) == 0:
                    results.append(
                        pool.apply_async(
                            helper.download_to_file,
                            args=(remote_path, local_url),
                            kwds={"overwrite_existing": overwrite, "skip_zero_size_check": skip_zero_size_check},
                        )
                    )
            for res in results:
                res.wait()

        return local_folder

    @classmethod
    def list(cls, remote_url, return_full_path=False):
        # type: (str, bool) -> Optional[List[str]]
        """
        Return a list of object names inside the base path

        :param str remote_url: The base path.
            For Google Storage, Azure and S3 it is the bucket of the path, for local files it is the root directory.
            For example: AWS S3: `s3://bucket/folder_` will list all the files you have in
            `s3://bucket-name/folder_*/*`. The same behaviour with Google Storage: `gs://bucket/folder_`,
            Azure blob storage: `azure://bucket/folder_` and also file system listing: `/mnt/share/folder_`
        :param bool return_full_path: If True, return a list of full object paths, otherwise return a list of
            relative object paths (default False).

        :return: The paths of all the objects in the storage base path under prefix, relative to the base path.
            None in case of list operation is not supported (http and https protocols for example)
        """
        helper = StorageHelper.get(remote_url)
        try:
            names_list = helper.list(prefix=remote_url)
        except Exception as ex:
            LoggerRoot.get_base_logger().warning("Can not list files for '{}' - {}".format(remote_url, ex))
            names_list = None
        return ["{}/{}".format(helper.base_url, name) for name in names_list] if return_full_path else names_list


# clearml/storage/util.py

import hashlib
import json
import re
import sys
from zlib import crc32
from typing import Optional, Union, Sequence, Dict
from pathlib2 import Path

from six.moves.urllib.parse import quote, urlparse, urlunparse
import six
import fnmatch

from ..debugging.log import LoggerRoot


def get_config_object_matcher(**patterns):
    unsupported = {k: v for k, v in patterns.items() if not isinstance(v, six.string_types)}
    if unsupported:
        raise ValueError('Unsupported object matcher (expecting string): %s'
                         % ', '.join('%s=%s' % (k, v) for k, v in unsupported.items()))

    # optimize simple patters
    starts_with = {k: v.rstrip('*') for k, v in patterns.items() if '*' not in v.rstrip('*') and '?' not in v}
    patterns = {k: v for k, v in patterns.items() if v not in starts_with}

    def _matcher(**kwargs):
        for key, value in kwargs.items():
            if not value:
                continue
            start = starts_with.get(key)
            if start:
                if value.startswith(start):
                    return True
            else:
                pat = patterns.get(key)
                if pat and fnmatch.fnmatch(value, pat):
                    return True

    return _matcher


def quote_url(url):
    parsed = urlparse(url)
    if parsed.scheme not in ('http', 'https'):
        return url
    parsed = parsed._replace(path=quote(parsed.path))
    return urlunparse(parsed)


def encode_string_to_filename(text):
    return quote(text, safe=" ")


def sha256sum(filename, skip_header=0, block_size=65536):
    # type: (str, int, int) -> (Optional[str], Optional[str])
    # create sha2 of the file, notice we skip the header of the file (32 bytes)
    # because sometimes that is the only change
    h = hashlib.sha256()
    file_hash = hashlib.sha256()
    b = bytearray(block_size)
    mv = memoryview(b)
    try:
        with open(filename, 'rb', buffering=0) as f:
            # skip header
            if skip_header:
                file_hash.update(f.read(skip_header))
            # noinspection PyUnresolvedReferences
            for n in iter(lambda: f.readinto(mv), 0):
                h.update(mv[:n])
                if skip_header:
                    file_hash.update(mv[:n])
    except Exception as e:
        LoggerRoot.get_base_logger().warning(str(e))
        return None, None

    return h.hexdigest(), file_hash.hexdigest() if skip_header else None


def md5text(text, seed=1337):
    # type: (str, Union[int, str]) -> str
    """
    Return md5 hash of a string
    Do not use this hash for security, if needed use something stronger like SHA2

    :param text: string to hash
    :param seed: use prefix seed for hashing
    :return: md5 string
    """
    return hash_text(text=text, seed=seed, hash_func='md5')


def crc32text(text, seed=1337):
    # type: (str, Union[int, str]) -> str
    """
    Return crc32 hash of a string
    Do not use this hash for security, if needed use something stronger like SHA2

    :param text: string to hash
    :param seed: use prefix seed for hashing
    :return: crc32 hex in string (32bits = 8 characters in hex)
    """
    return '{:08x}'.format(crc32((str(seed)+str(text)).encode('utf-8')))


def hash_text(text, seed=1337, hash_func='md5'):
    # type: (str, Union[int, str], str) -> str
    """
    Return hash_func (md5/sha1/sha256/sha384/sha512) hash of a string

    :param text: string to hash
    :param seed: use prefix seed for hashing
    :param hash_func: hashing function. currently supported md5 sha256
    :return: hashed string
    """
    assert hash_func in ('md5', 'sha256', 'sha256', 'sha384', 'sha512')
    h = getattr(hashlib, hash_func)()
    h.update((str(seed) + str(text)).encode('utf-8'))
    return h.hexdigest()


def hash_dict(a_dict, seed=1337, hash_func='md5'):
    # type: (Dict, Union[int, str], str) -> str
    """
    Return hash_func (crc32/md5/sha1/sha256/sha384/sha512) hash of the dict values
    (dict must be JSON serializable)

    :param a_dict: a dictionary to hash
    :param seed: use prefix seed for hashing
    :param hash_func: hashing function. currently supported md5 sha256
    :return: hashed string
    """
    assert hash_func in ('crc32', 'md5', 'sha256', 'sha256', 'sha384', 'sha512')
    repr_string = json.dumps(a_dict, sort_keys=True)
    if hash_func == 'crc32':
        return crc32text(repr_string, seed=seed)
    else:
        return hash_text(repr_string, seed=seed, hash_func=hash_func)


def is_windows():
    """
    :return: True if currently running on windows OS
    """
    return sys.platform == 'win32'


def format_size(size_in_bytes, binary=False):
    # type: (Union[int, float], bool) -> str
    """
    Return the size in human readable format (string)
    Matching humanfriendly.format_size outputs

    :param size_in_bytes: number of bytes
    :param binary: If `True` 1 Kb equals 1024 bytes, if False (default) 1 KB = 1000 bytes
    :return: string representation of the number of bytes (b,Kb,Mb,Gb, Tb,)
        >>> format_size(0)
        '0 bytes'
        >>> format_size(1)
        '1 byte'
        >>> format_size(5)
        '5 bytes'
        > format_size(1000)
        '1 KB'
        > format_size(1024, binary=True)
        '1 KiB'
        >>> format_size(1000 ** 3 * 4)
        '4 GB'
    """
    size = float(size_in_bytes)
    # single byte is the exception here
    if size == 1:
        return '{} byte'.format(int(size))
    k = 1024 if binary else 1000
    scale = ('bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB') if binary else ('bytes', 'KB', 'MB', 'GB', 'TB', 'PB')
    for i, m in enumerate(scale):
        if size < k**(i+1) or i == len(scale)-1:
            return ('{:.2f}'.format(size/(k**i)).rstrip('0').rstrip('.')
                    if i > 0 else '{}'.format(int(size))) + ' ' + m
    # we should never get here
    return '{} {}'.format(int(size), scale[0])


def parse_size(size, binary=False):
    """
    Parse a human readable data size and return the number of bytes.
    Match humanfriendly.parse_size

    :param size: The human readable file size to parse (a string).
    :param binary: :data:`True` to use binary multiples of bytes (base-2) for
                   ambiguous unit symbols and names, :data:`False` to use
                   decimal multiples of bytes (base-10).
    :returns: The corresponding size in bytes (an integer).
    :raises: :exc:`InvalidSize` when the input can't be parsed.

    This function knows how to parse sizes in bytes, kilobytes, megabytes,
    gigabytes, terabytes and petabytes. Some examples:
        >>> parse_size('42')
        42
        >>> parse_size('13b')
        13
        >>> parse_size('5 bytes')
        5
        >>> parse_size('1 KB')
        1000
        >>> parse_size('1 kilobyte')
        1000
        >>> parse_size('1 KiB')
        1024
        >>> parse_size('1 KB', binary=True)
        1024
        >>> parse_size('1.5 GB')
        1500000000
        >>> parse_size('1.5 GB', binary=True)
        1610612736
    """
    def tokenize(text):
        tokenized_input = []
        for token in re.split(r'(\d+(?:\.\d+)?)', text):
            token = token.strip()
            if re.match(r'\d+\.\d+', token):
                tokenized_input.append(float(token))
            elif token.isdigit():
                tokenized_input.append(int(token))
            elif token:
                tokenized_input.append(token)
        return tokenized_input
    tokens = tokenize(str(size))
    if tokens and isinstance(tokens[0], (int, float)):
        disk_size_units_b = \
            (('B', 'bytes'), ('KiB', 'kibibyte'), ('MiB', 'mebibyte'), ('GiB', 'gibibyte'),
             ('TiB', 'tebibyte'), ('PiB', 'pebibyte'))
        disk_size_units_d = \
            (('B', 'bytes'), ('KB', 'kilobyte'), ('MB', 'megabyte'), ('GB', 'gigabyte'),
             ('TB', 'terabyte'), ('PB', 'petabyte'))
        disk_size_units_b = [(1024 ** i, s[0], s[1]) for i, s in enumerate(disk_size_units_b)]
        k = 1024 if binary else 1000
        disk_size_units_d = [(k ** i, s[0], s[1]) for i, s in enumerate(disk_size_units_d)]
        disk_size_units = (disk_size_units_b + disk_size_units_d) \
            if binary else (disk_size_units_d + disk_size_units_b)

        # Get the normalized unit (if any) from the tokenized input.
        normalized_unit = tokens[1].lower() if len(tokens) == 2 and isinstance(tokens[1], str) else ''
        # If the input contains only a number, it's assumed to be the number of
        # bytes. The second token can also explicitly reference the unit bytes.
        if len(tokens) == 1 or normalized_unit.startswith('b'):
            return int(tokens[0])
        # Otherwise we expect two tokens: A number and a unit.
        if normalized_unit:
            # Convert plural units to singular units, for details:
            # https://github.com/xolox/python-humanfriendly/issues/26
            normalized_unit = normalized_unit.rstrip('s')
            for k, low, high in disk_size_units:
                # First we check for unambiguous symbols (KiB, MiB, GiB, etc)
                # and names (kibibyte, mebibyte, gibibyte, etc) because their
                # handling is always the same.
                if normalized_unit in (low.lower(), high.lower()):
                    return int(tokens[0] * k)
                # Now we will deal with ambiguous prefixes (K, M, G, etc),
                # symbols (KB, MB, GB, etc) and names (kilobyte, megabyte,
                # gigabyte, etc) according to the caller's preference.
                if (normalized_unit in (low.lower(), high.lower()) or
                        normalized_unit.startswith(low.lower())):
                    return int(tokens[0] * k)

    raise ValueError("Failed to parse size! (input {} was tokenized as {})".format(size, tokens))


def get_common_path(list_of_files):
    # type: (Sequence[Union[str, Path]]) -> Optional[str]
    """
    Return the common path of a list of files

    :param list_of_files: list of files (str or Path objects)
    :return: Common path string (always absolute) or None if common path could not be found
    """
    if not list_of_files:
        return None

    # a single file has its parent as common path
    if len(list_of_files) == 1:
        return Path(list_of_files[0]).absolute().parent.as_posix()

    # find common path to support folder structure inside zip
    common_path_parts = Path(list_of_files[0]).absolute().parts
    for f in list_of_files:
        f_parts = Path(f).absolute().parts
        num_p = min(len(f_parts), len(common_path_parts))
        if f_parts[:num_p] == common_path_parts[:num_p]:
            common_path_parts = common_path_parts[:num_p]
            continue
        num_p = min(
            [i for i, (a, b) in enumerate(zip(common_path_parts[:num_p], f_parts[:num_p])) if a != b] or [-1])
        # no common path, break
        if num_p < 0:
            common_path_parts = []
            break
        # update common path
        common_path_parts = common_path_parts[:num_p]

    if common_path_parts:
        common_path = Path()
        for f in common_path_parts:
            common_path /= f
        return common_path.as_posix()

    return None


# clearml/utilities/__init__.py

from .dicts import Logs

__all__ = ["Logs"]


# clearml/utilities/async_manager.py

import os
import time
import six

from .process.mp import SingletonLock


class AsyncManagerMixin(object):
    _async_results_lock = SingletonLock()
    # per pid (process) list of async jobs (support for sub-processes forking)
    _async_results = {}

    @classmethod
    def _add_async_result(cls, result, wait_on_max_results=None, wait_time=30, wait_cb=None):
        while True:
            try:
                cls._async_results_lock.acquire()
                # discard completed results
                pid = os.getpid()
                cls._async_results[pid] = [r for r in cls._async_results.get(pid, []) if not r.ready()]
                num_results = len(cls._async_results[pid])
                if wait_on_max_results is not None and num_results >= wait_on_max_results:
                    # At least max_results results are still pending, wait
                    if wait_cb:
                        wait_cb(num_results)
                    if wait_time:
                        time.sleep(wait_time)
                    continue
                # add result
                if result and not result.ready():
                    if not cls._async_results.get(pid):
                        cls._async_results[pid] = []
                    cls._async_results[pid].append(result)
                break
            finally:
                cls._async_results_lock.release()

    @classmethod
    def wait_for_results(cls, timeout=None, max_num_uploads=None):
        remaining = timeout
        count = 0
        pid = os.getpid()
        for r in cls._async_results.get(pid, []):
            if r.ready():
                continue
            t = time.time()
            # bugfix for python2.7 threading issues
            if six.PY2 and not remaining:
                while not r.ready():
                    r.wait(timeout=2.)
            else:
                r.wait(timeout=remaining)
            count += 1
            if max_num_uploads is not None and max_num_uploads - count <= 0:
                break
            if timeout is not None:
                remaining = max(0., remaining - max(0., time.time() - t))
                if not remaining:
                    break

    @classmethod
    def get_num_results(cls):
        pid = os.getpid()
        if cls._async_results.get(pid, []):
            return len([r for r in cls._async_results.get(pid, []) if not r.ready()])
        else:
            return 0


# clearml/utilities/attrs.py

import attr

from .version import Version


class attrs(object):
    def __init__(self, *args, **kwargs):
        if any(x in kwargs for x in ("eq", "order")):
            raise RuntimeError("Only `cmp` is supported for attr.attrs, not `eq` or `order`")
        if Version(attr.__version__) >= Version("19.2"):
            cmp = kwargs.pop("cmp", None)
            if cmp is not None:
                kwargs["eq"] = kwargs["order"] = cmp
        self.args = args
        self.kwargs = kwargs

    def __call__(self, f):
        return attr.attrs(*self.args, **self.kwargs)(f)


# clearml/utilities/check_updates.py

from __future__ import absolute_import, division, print_function

import json
import os

import requests

from ..backend_api.session import Session
from ..backend_config import EnvEntry

from .version import Version


class CheckPackageUpdates(object):
    _package_version_checked = False

    @classmethod
    def check_new_package_available(cls, only_once=False):
        """
        :return: True, if there is a newer package in PyPI.
        """
        if only_once and cls._package_version_checked:
            return None

        # noinspection PyBroadException
        try:
            cls._package_version_checked = True
            client, version = Session._client[0]
            version = Version(version)
            is_demo = 'https://demoapi.demo.clear.ml/'.startswith(Session.get_api_server_host())

            update_server_releases = requests.get(
                'https://updates.clear.ml/updates',
                json={"demo": is_demo,
                      "versions": {c: str(v) for c, v in Session._client},
                      "CI": str(os.environ.get('CI', ''))},
                timeout=3.0
            )

            if update_server_releases.ok:
                update_server_releases = update_server_releases.json()
            else:
                return None

            client_answer = update_server_releases.get(client, {})
            if "version" not in client_answer:
                return None

            # do not output upgrade message if we are running inside a CI process.
            if EnvEntry("CI", type=bool, ignore_errors=True).get():
                return None

            latest_version = Version(client_answer["version"])

            if version >= latest_version:
                return None
            not_patch_upgrade = latest_version.release[:2] == version.release[:2]
            return str(latest_version), not_patch_upgrade, client_answer.get("description").split("\r\n")
        except Exception:
            return None

    @staticmethod
    def get_version_from_updates_server(cur_version):
        """
        Get the latest version for clearml from updates server
        :param cur_version: The current running version of clearml
        :type cur_version: Version
        """
        try:
            _ = requests.get('https://updates.clear.ml/updates',
                             data=json.dumps({"versions": {"clearml": str(cur_version)}}),
                             timeout=1.0)
            return
        except Exception:
            pass


# clearml/utilities/config.py

from __future__ import division

import json

import six
import pyparsing

from .dicts import hocon_quote_key, hocon_unquote_key
from .pyhocon import ConfigFactory, HOCONConverter
from ..storage.util import parse_size


def parse_human_size(value):
    if isinstance(value, six.string_types):
        return parse_size(value)
    return value


def get_percentage(config, key, required=True, default=None):
    if required:
        value = config.get(key)
    else:
        value = config.get(key, default)
        if value is None:
            return
    try:
        if isinstance(value, six.string_types):
            value = value.strip()
            if value.endswith('%'):
                # "50%" => 0.5
                return float(value.strip('%')) / 100.
            # "50" => 50

        value = float(value)
        if value < 1:
            # 0.5 => 50% => 0.5
            return value

        # 50 => 0.5, 10.5 => 0.105
        return value / 100.

    except ValueError as e:
        raise ValueError('Config: failed parsing %s: %s' % (key, e))


def get_human_size_default(config, key, default=None):
    raw_value = config.get(key, default)

    if raw_value is None:
        return default

    return parse_human_size(raw_value)


def config_dict_to_text(config):
    # if already string return as is
    if isinstance(config, six.string_types):
        return config
    if not isinstance(config, (dict, list)):
        raise ValueError("Configuration only supports dictionary/list objects")
    try:
        # noinspection PyBroadException
        try:
            text = HOCONConverter.to_hocon(ConfigFactory.from_dict(hocon_quote_key(config)))
        except Exception:
            # fallback json+pyhocon
            # hack, pyhocon is not very good with dict conversion so we pass through json
            import json
            text = json.dumps(config)
            text = HOCONConverter.to_hocon(ConfigFactory.parse_string(text))

    except Exception:
        raise ValueError("Could not serialize configuration dictionary:\n", config)
    return text


def text_to_config_dict(text):
    if not isinstance(text, six.string_types):
        raise ValueError("Configuration parsing only supports string")
    # noinspection PyBroadException
    try:
        return hocon_unquote_key(ConfigFactory.parse_string(text))
    except pyparsing.ParseBaseException as ex:
        pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column)
        six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None)
    except Exception:
        six.raise_from(ValueError("Could not parse configuration text:\n{}".format(text)), None)


def verify_basic_value(value):
    # return True if value of of basic type (json serializable)
    if not isinstance(value,
                      six.string_types + six.integer_types +
                      (six.text_type, float, list, tuple, dict, type(None))):
        return False
    try:
        json.dumps(value)
        return True
    except TypeError:
        return False


# clearml/utilities/deferred.py

import threading
from functools import wraps

import attr
import six


class DeferredExecutionPool(object):
    @attr.s
    class _DeferredAction(object):
        method = attr.ib()
        args = attr.ib()
        kwargs = attr.ib()

    def __init__(self, instance):
        self._instance = instance
        self._pool = []
        self._lock = threading.Lock()

    def add(self, callable_, *args, **kwargs):
        self._pool.append(self._DeferredAction(callable_, args, kwargs))

    def clear(self):
        with self._lock:
            pool = self._pool
            self._pool = []
            return pool

    def apply(self):
        pool = self.clear()
        for action in pool:
            action.method(self._instance, *action.args, **action.kwargs)

    def copy_from(self, other):
        if not isinstance(self._instance, type(other._instance)):
            raise ValueError("Copy deferred actions must be with the same instance type")

        self._pool = other._pool[:]


class ParameterizedDefaultDict(dict):
    def __init__(self, factory, *args, **kwargs):
        super(ParameterizedDefaultDict, self).__init__(*args, **kwargs)
        self._factory = factory

    def __missing__(self, key):
        self[key] = self._factory(key)
        return self[key]


class DeferredExecution(object):
    def __init__(self, pool_cls=DeferredExecutionPool):
        self._pools = ParameterizedDefaultDict(pool_cls)

    def __get__(self, instance, owner):
        if not instance:
            return self

        return self._pools[instance]

    def defer_execution(self, condition_or_attr_name=True):
        """
        Deferred execution decorator, designed to wrap class functions for classes containing a deferred execution pool.
        :param condition_or_attr_name: Condition controlling whether wrapped function should be deferred.
            True by default. If a callable is provided, it will be called with the class instance (self)
            as first argument. If a string is provided, a class instance (self) attribute by that name is evaluated.
        :return:
        """
        def decorator(func):
            @wraps(func)
            def wrapper(instance, *args, **kwargs):
                if self._resolve_condition(instance, condition_or_attr_name):
                    self._pools[instance].add(func, *args, **kwargs)
                else:
                    return func(instance, *args, **kwargs)
            return wrapper
        return decorator

    @staticmethod
    def _resolve_condition(instance, condition_or_attr_name):
        if callable(condition_or_attr_name):
            return condition_or_attr_name(instance)
        elif isinstance(condition_or_attr_name, six.string_types):
            return getattr(instance, condition_or_attr_name)
        return condition_or_attr_name

    def _apply(self, instance, condition_or_attr_name):
        if self._resolve_condition(instance, condition_or_attr_name):
            self._pools[instance].apply()

    def apply_after(self, condition_or_attr_name=True):
        """
        Decorator for applying deferred execution pool after wrapped function has completed
        :param condition_or_attr_name: Condition controlling whether deferred pool should be applied. True by default.
            If a callable is provided, it will be called with the class instance (self) as first argument.
            If a string is provided, a class instance (self) attribute by that name is evaluated.
        """
        def decorator(func):
            @wraps(func)
            def wrapper(instance, *args, **kwargs):
                res = func(instance, *args, **kwargs)
                self._apply(instance, condition_or_attr_name)
                return res
            return wrapper
        return decorator

    def apply_before(self, condition_or_attr_name=True):
        """
        Decorator for applying deferred execution pool before wrapped function is executed
        :param condition_or_attr_name: Condition controlling whether deferred pool should be applied. True by default.
            If a callable is provided, it will be called with the class instance (self) as first argument.
            If a string is provided, a class instance (self) attribute by that name is evaluated.
        """
        def decorator(func):
            @wraps(func)
            def wrapper(instance, *args, **kwargs):
                self._apply(instance, condition_or_attr_name)
                return func(instance, *args, **kwargs)
            return wrapper
        return decorator


# clearml/utilities/dicts.py

""" Utilities """
from typing import Optional, Any

_epsilon = 0.00001


class ReadOnlyDict(dict):
    def __readonly__(self, *args, **kwargs):
        raise ValueError("This is a read only dictionary")
    __setitem__ = __readonly__
    __delitem__ = __readonly__
    pop = __readonly__
    popitem = __readonly__
    clear = __readonly__
    update = __readonly__
    setdefault = __readonly__
    del __readonly__


class Logs:
    _logs_instances = []

    def __init__(self, data=None):
        self._data = data or {}
        self._logs_instances.append(self)

    def reset(self):
        self._data = {}

    @property
    def data(self):
        return self._data

    @classmethod
    def get_instances(cls):
        return cls._logs_instances


class BlobsDict(dict):
    """
    Overloading getitem so that the 'data' copy is only done when the dictionary item is accessed.
    """

    def __init__(self, *args, **kwargs):
        super(BlobsDict, self).__init__(*args, **kwargs)

    def __getitem__(self, k):
        val = super(BlobsDict, self).__getitem__(k)
        if isinstance(val, dict):
            return BlobsDict(val)
        # We need to ask isinstance without actually importing blob here
        # so we accept that in order to appreciate beauty in life we must have a dash of ugliness.
        # ans instead of -
        # elif isinstance(val, Blob):
        # we ask:
        elif hasattr(val, '__class__') and val.__class__.__name__ == 'Blob':
            return val.data
        else:
            return val


class NestedBlobsDict(BlobsDict):
    """A dictionary that applies an arbitrary key-altering function
       before accessing the keys."""

    def __init__(self, *args, **kwargs):
        super(NestedBlobsDict, self).__init__(*args, **kwargs)

    def __getitem__(self, keys_str=''):

        if keys_str == '':
            return super(NestedBlobsDict, self).__getitem__(self)

        keylist = keys_str.split('.')

        cur = super(NestedBlobsDict, self).__getitem__(keylist[0])
        if len(keylist) == 1:
            return cur
        else:
            return NestedBlobsDict(cur)['.'.join(keylist[1:])]

    def __contains__(self, keys_str):
        keylist = self.keys()
        return keys_str in keylist

    def as_dict(self):
        return dict(self)

    def get(self, keys_str, default=None):
        # noinspection PyBroadException
        try:
            return self[keys_str]
        except Exception:
            return None

    def _keys(self, cur_dict, path):
        deep_keys = []
        cur_keys = dict.keys(cur_dict)

        for key in cur_keys:
            if isinstance(cur_dict[key], dict):
                if len(path) > 0:
                    deep_keys.extend(self._keys(cur_dict[key], path + '.' + key))
                else:
                    deep_keys.extend(self._keys(cur_dict[key], key))
            else:
                if len(path) > 0:
                    deep_keys.append(path + '.' + key)
                else:
                    deep_keys.append(key)

        return deep_keys

    def keys(self):
        return self._keys(self, '')


def merge_dicts(dict1, dict2):
    """ Recursively merges dict2 into dict1 """
    if not isinstance(dict1, dict) or not isinstance(dict2, dict):
        return dict2
    for k in dict2:
        if k in dict1:
            dict1[k] = merge_dicts(dict1[k], dict2[k])
        else:
            dict1[k] = dict2[k]
    return dict1


def hocon_quote_key(a_obj):
    """ Recursively quote key with '.' to \"key\" """
    if isinstance(a_obj, list):
        return [hocon_quote_key(a) for a in a_obj]
    elif isinstance(a_obj, tuple):
        return tuple(hocon_quote_key(a) for a in a_obj)
    elif not isinstance(a_obj, dict):
        return a_obj

    # preserve dict type
    a_dict = a_obj
    new_dict = type(a_dict)()
    for k, v in a_dict.items():
        if isinstance(k, str) and '.' in k:
            new_dict['"{}"'.format(k)] = hocon_quote_key(v)
        else:
            new_dict[k] = hocon_quote_key(v)
    return new_dict


def hocon_unquote_key(a_obj):
    """ Recursively unquote \"key\" with '.' to key """

    if isinstance(a_obj, list):
        return [hocon_unquote_key(a) for a in a_obj]
    elif isinstance(a_obj, tuple):
        return tuple(hocon_unquote_key(a) for a in a_obj)
    elif not isinstance(a_obj, dict):
        return a_obj

    a_dict = a_obj

    # ConfigTree to dict
    if hasattr(a_dict, 'as_plain_ordered_dict'):
        a_dict = a_dict.as_plain_ordered_dict()

    # preserve dict type
    new_dict = type(a_dict)()
    for k, v in a_dict.items():
        if isinstance(k, str) and k[0] == '"' and k[-1] == '"' and '.' in k:
            new_dict[k[1:-1]] = hocon_unquote_key(v)
        else:
            new_dict[k] = hocon_unquote_key(v)
    return new_dict


def cast_str_to_bool(value, strip=True):
    # type: (Any, bool) -> Optional[bool]
    a_strip_v = value if not strip else str(value).lower().strip()
    if a_strip_v == 'false' or not a_strip_v:
        return False
    elif a_strip_v == 'true':
        return True
    else:
        # first try to cast to integer
        try:
            return bool(int(a_strip_v))
        except (ValueError, TypeError):
            return None


# clearml/utilities/enum.py

""" Enum utilities """


class EnumOptions(object):
    """ Base class for enum-like classes using class-attributes with string values to represent enum key/value pairs """
    __cache = None

    @classmethod
    def values(cls):
        """ Extract list of enum-like options based on the derived classes' attributes.
            Any class attribute who's key doesn't start with an underscore and who's value is not a class method
             or callable is considered an option.
            Returns a list of attribute names representing the options.
        """
        if cls.__cache is None:
            cls.__cache = [v for k, v in vars(cls).items() if
                           not k.startswith('_') and not callable(v) and not isinstance(v, classmethod)]
        return cls.__cache


class Options(object):
    @classmethod
    def _all(cls):
        return {k: v for k, v in vars(cls) if not k.startswith('_')}


# clearml/utilities/io_manager.py

class IOCallsManager(object):

    def __init__(self):
        self.threads_io = {}

    def add_io_to_thread(self, thread_id, io_object):
        if thread_id in self.threads_io:
            self.threads_io[thread_id].add(id(io_object))
        else:
            self.threads_io[thread_id] = {id(io_object)}
        if self._io_has_canvas_figure(io_object):
            self.threads_io[thread_id].add(id(io_object.canvas.figure))

    def is_plot_called(self, thread_id, io_object):
        return id(io_object) in self.threads_io.get(thread_id, set())

    def remove_io_to_thread(self, thread_id, io_object):
        try:
            self.threads_io[thread_id].remove(id(io_object))
            if self._io_has_canvas_figure(io_object):
                self.threads_io[thread_id].remove(id(io_object.canvas.figure))
        except Exception:
            pass

    def remove_thread(self, thread_id):
        if thread_id in self.threads_io:
            del self.threads_io[thread_id]

    @staticmethod
    def _io_has_canvas_figure(io_object):
        return hasattr(io_object, 'canvas') and hasattr(io_object.canvas, 'figure')


# clearml/utilities/plotly_reporter.py

import json
import numpy as np

from ..errors import UsageError
from ..utilities.dicts import merge_dicts

try:
    import pandas as pd
except ImportError:
    pd = None
from attr import attrs, attrib


def create_2d_histogram_plot(np_row_wise, labels, title=None, xtitle=None, ytitle=None, series=None, xlabels=None,
                             comment=None, mode='group', layout_config=None):
    """
    Create a 2D Plotly histogram chart from a 2D numpy array
    :param np_row_wise: 2D numpy data array
    :param labels: Histogram labels
    :param title: Chart title
    :param xtitle: X-Series title
    :param ytitle: Y-Series title
    :param comment: comment underneath the title
    :param mode: multiple histograms mode. valid options are: stack / group / relative. Default is 'group'.
    :param layout_config: optional extra layout configuration
    :return: Plotly chart dict.
    """
    assert mode in ('stack', 'group', 'relative')

    np_row_wise = np.atleast_2d(np_row_wise)
    assert len(np_row_wise.shape) == 2, "Expected a 2D numpy array"
    use_series = bool(labels)

    # using labels without xlabels leads to original behavior
    if labels is not None and xlabels is None:
        assert len(labels) == np_row_wise.shape[0], "Please provide a label for each data row"
    elif xlabels is None:
        fake_label = series or ''
        labels = [fake_label] * np_row_wise.shape[0]
    elif labels:
        if len(labels) == 1:
            labels = [labels] * np_row_wise.shape[0]
        assert len(xlabels) == np_row_wise.shape[1]
    elif not labels and xlabels:
        labels = [series]

    data = [_np_row_to_plotly_data_item(np_row=np_row_wise[i, :], label=labels[i] if labels else None, xlabels=xlabels)
            for i in range(np_row_wise.shape[0])]
    return _plotly_hist_dict(title=series if use_series else title,
                             xtitle=xtitle, ytitle=ytitle, mode=mode, data=data, comment=comment,
                             layout_config=layout_config)


def _to_np_array(value):
    if not isinstance(value, np.ndarray):
        value = np.array(value)

    return value


@attrs
class SeriesInfo(object):
    name = attrib(type=str)
    data = attrib(type=np.ndarray, converter=_to_np_array)
    labels = attrib(default=None)

    @data.validator
    def _validate_data(self, _, value):
        if value.ndim != 2:
            raise ValueError("Expected series data to be 2D numpy array")

        if value.shape[1] != 2:
            raise ValueError("Expected series data to have 2 columns")

    def __attrs_post_init__(self):
        if (self.labels is not None) and (len(self.labels) != self.data.shape[0]):
            raise ValueError(
                "If 'labels' is provided, it must be a list or tuple, "
                "the same length as the data"
            )


def create_line_plot(title, series, xtitle, ytitle, mode='lines', reverse_xaxis=False,
                     comment=None, MAX_SIZE=None, layout_config=None):
    plotly_obj = _plotly_scatter_layout_dict(
        title=title if not comment else (title + '<br><sup>' + comment + '</sup>'),
        xaxis_title=xtitle,
        yaxis_title=ytitle,
    )

    if reverse_xaxis:
        plotly_obj["layout"]["xaxis"]["autorange"] = "reversed"

    # check maximum size of data
    MAX_SIZE = MAX_SIZE or 800000
    series_sizes = [s.data.size for s in series]
    total_size = sum(series_sizes)
    if total_size > MAX_SIZE:
        # we need to downscale
        base_size = MAX_SIZE / len(series_sizes)
        baseused_size = sum([min(s, base_size) for s in series_sizes])
        leftover = MAX_SIZE - baseused_size
        for s in series:
            # if we need to down-sample, use low-pass average filter and sampling
            if s.data.size >= base_size:
                budget = base_size
                # if we have some leftover in the budget, split based on series sizes
                if leftover > 0:
                    # calculate the relative overflow  of this series compared to all the overflows
                    # then multiply it by the leftover budget
                    budget += int(leftover * s.data.size / (total_size - baseused_size))
                step = int(np.ceil(s.data.size / float(budget)))
                x = s.data[:, 0][::-step][::-1]
                y = s.data[:, 1]
                y_low_pass = np.convolve(y, np.ones(shape=(step,), dtype=y.dtype) / float(step), mode='same')
                y = y_low_pass[::-step][::-1]
                s.data = np.array([x, y], dtype=s.data.dtype).T

            # decide on number of points between mean and max
            s_max = np.max(np.abs(s.data), axis=0)
            s_max = np.maximum(s_max, s_max * 0 + 0.01)
            digits = np.maximum(np.array([1, 1]), np.array([6, 6]) - np.floor(np.abs(np.log10(s_max))))
            s.data[:, 0] = np.round(s.data[:, 0], int(digits[0]))
            s.data[:, 1] = np.round(s.data[:, 1], int(digits[1]))

    plotly_obj["data"].extend({
        "name": s.name,
        "x": s.data[:, 0].tolist(),
        "y": s.data[:, 1].tolist(),
        "mode": mode,
        "text": s.labels,
        "type": "scatter",
    } for s in series)

    if layout_config:
        plotly_obj["layout"] = merge_dicts(plotly_obj["layout"], layout_config)

    return plotly_obj


def create_2d_scatter_series(np_row_wise, title="Scatter", series_name="Series", xtitle="x", ytitle="y", mode="lines",
                             labels=None, comment=None, layout_config=None):
    """
    Create a 2D scatter Plotly graph from a 2 column numpy array
    :param np_row_wise: 2 column numpy data array [(x0,y0), (x1,y1) ...]
    :param title: Chart title
    :param series_name: Series name
    :param xtitle: X-axis title
    :param ytitle: Y-axis title
    :param mode: scatter type mode ('lines' / 'markers' / 'lines+markers')
    :param labels: label (text) per point on the scatter graph
    :param comment: comment underneath the title
    :param layout_config: optional dictionary for layout configuration, passed directly to plotly
    :return: Plotly chart dict.
    """
    plotly_obj = _plotly_scatter_layout_dict(  # noqa: F841
        title=title, xaxis_title=xtitle, yaxis_title=ytitle, comment=comment)
    assert np_row_wise.ndim == 2, "Expected a 2D numpy array"
    assert np_row_wise.shape[1] == 2, "Expected two columns X/Y e.g. [(x0,y0), (x1,y1) ...]"

    # this_scatter_data = {
    #     "name": series_name,
    #     "x": np_row_wise[:, 0].tolist(),
    #     "y": np_row_wise[:, 1].tolist(),
    #     "mode": mode,
    #     "text": labels,
    #     "type": "scatter"
    # }
    # plotly_obj["data"].append(this_scatter_data)
    # return plotly_obj
    series = SeriesInfo(name=series_name, data=np_row_wise, labels=labels)

    return create_line_plot(title=title, series=[series], xtitle=xtitle, ytitle=ytitle, mode=mode,
                            comment=comment, MAX_SIZE=100000, layout_config=layout_config)


def create_3d_scatter_series(np_row_wise, title="Scatter", series_name="Series", xtitle="x", ytitle="y", ztitle="z",
                             mode="lines", color=((217, 217, 217, 0.14),), marker_size=5, line_width=0.8,
                             labels=None, fill_axis=-1, plotly_obj=None, layout_config=None):
    """
    Create a 3D scatter Plotly graph from a 3 column numpy array
    :param np_row_wise: 3 column numpy data array [(x0,y0,z0), (x1,y1,z1) ...]
    :param title: Chart title
    :param series_name: Series name
    :param xtitle: X-axis title
    :param ytitle: Y-axis title
    :param ztitle: Z-axis title
    :param labels: label (text) per point on the scatter graph
    :param fill_axis: fill area under the curve
    :param layout_config: additional layout configuration
    :return: Plotly chart dict.
    """
    if not plotly_obj:
        plotly_obj = plotly_scatter3d_layout_dict(
            title=title, xaxis_title=xtitle, yaxis_title=ytitle, zaxis_title=ztitle)
    assert np_row_wise.ndim == 2, "Expected a 2D numpy array"
    assert np_row_wise.shape[1] == 3, "Expected three columns X/Y/Z e.g. [(x0,y0,z0), (x1,y1,z1) ...]"

    c = color[0]
    c = (int(c[0]), int(c[1]), int(c[2]), float(c[3]))
    this_scatter_data = {
        "name": series_name,
        "x": np_row_wise[:, 0].tolist(),
        "y": np_row_wise[:, 1].tolist(),
        "z": np_row_wise[:, 2].tolist(),
        "text": labels,
        "type": "scatter3d",
        "mode": mode,
        'marker': {
            'size': marker_size,
            'line': {
                'color': 'rgba(%d, %d, %d, %f.2)' % (c[0], c[1], c[2], c[3]),
                'width': line_width
            },
            'opacity': 0.8
        },
    }
    plotly_obj["data"].append(this_scatter_data)

    if layout_config:
        plotly_obj["layout"] = merge_dicts(plotly_obj["layout"], layout_config)

    return plotly_obj


def create_value_matrix(np_value_matrix, title="Heatmap Matrix", xlabels=None, ylabels=None, xtitle="X", ytitle="Y",
                        custom_colors=True, series=None, comment=None, yaxis_reversed=False, layout_config=None):
    conf_matrix_plot = {
        "data": [
            {
                "x": xlabels,
                "y": ylabels,
                "z": np_value_matrix.tolist(),
                "type": "heatmap"
            }
        ],
        "layout": {
            "showlegend": True,
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),

            "xaxis": {
                "title": xtitle,
            },
            "yaxis": {
                "title": ytitle
            },
            "name": series,
        }
    }
    if yaxis_reversed:
        conf_matrix_plot['layout']['yaxis']['autorange'] = "reversed"

    if custom_colors and not layout_config:
        scale, bar = _get_z_colorbar_data()
        conf_matrix_plot["data"][0].update({"colorscale": scale})
        conf_matrix_plot["data"][0].update({"colorbar": bar})

    if layout_config:
        conf_matrix_plot["data"][0] = merge_dicts(conf_matrix_plot["data"][0], layout_config)
        conf_matrix_plot["layout"] = merge_dicts(conf_matrix_plot["layout"], layout_config)

    return conf_matrix_plot


def create_3d_surface(np_value_matrix, title="3D Surface", xlabels=None, ylabels=None, xtitle="X", ytitle="Y",
                      ztitle="Z", custom_colors=True, series=None, camera=None, comment=None, layout_config=None):
    surface_plot = {
        "data": [
            {
                "z": np_value_matrix.tolist(),
                "type": "surface",
                "contours": {
                    "y": {
                        "show": False,
                        "highlightcolor": "#fff4ff",
                        "project": {"y": True}
                    }
                },
                "showscale": False,
            }
        ],
        "layout": {
            "scene": {
                "xaxis": {
                    "title": xtitle,
                    "showgrid": False,
                    "nticks": 10,
                    "ticktext": xlabels,
                    "tickvals": list(range(len(xlabels))) if xlabels else None,
                },
                "yaxis": {
                    "title": ytitle,
                    "showgrid": False,
                    "nticks": 10,
                    "ticktext": ylabels,
                    "tickvals": list(range(len(ylabels))) if ylabels else ylabels,
                },
                "zaxis": {
                    "title": ztitle,
                    "nticks": 5,
                },
            },
            "showlegend": False,
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "name": series,
        }
    }
    if camera:
        surface_plot['layout']['scene']['camera'] = {"eye": {"x": camera[0], "y": camera[1], "z": camera[2]}}

    if custom_colors:
        scale, bar = _get_z_colorbar_data()
        surface_plot["data"][0].update({"colorscale": scale})
        surface_plot["data"][0].update({"colorbar": bar})

    if layout_config:
        surface_plot["layout"] = merge_dicts(surface_plot["layout"], layout_config)

    return surface_plot


def create_image_plot(image_src, title, width=640, height=480, series=None, comment=None, layout_config=None):
    image_plot = {
        "data": [],
        "layout": {
            "xaxis": {"visible": False, "range": [0, width]},
            "yaxis": {"visible": False, "range": [0, height], "scaleanchor": "x"},
            # "width": width,
            # "height": height,
            "margin": {'l': 0, 'r': 0, 't': 64, 'b': 0},
            "images": [{
                "sizex": width,
                "sizey": height,
                "xref": "x",
                "yref": "y",
                "opacity": 1.0,
                "x": 0,
                "y": height,
                # "xanchor": "left",
                # "yanchor": "bottom",
                "sizing": "contain",
                "layer": "below",
                "source": image_src
            }],
            "showlegend": False,
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "name": series,
        }
    }

    if layout_config:
        image_plot["layout"] = merge_dicts(image_plot["layout"], layout_config)

    return image_plot


def _get_z_colorbar_data(z_data=None, values=None, colors=None):
    if values is None:
        values = [0, 1. / 10, 2. / 10, 6. / 10, 9. / 10]
    if colors is None:
        colors = [(71, 17, 100), (53, 92, 140), (37, 139, 141), (66, 189, 112), (141, 314, 68), (221, 226, 24)]
    if z_data is not None:
        data = np.array(z_data)
        max_z = data.max()
        scaler = max_z
        values = [float(v * scaler) for v in values[0:5]]
    values.append(1.0)  # poltly quirk?
    # we do not want to show the first and last value
    tickvalues = [" %.3f " % v for v in values[1:]]
    tickvalues = [float(v) for v in tickvalues]
    # tickvalues.pop()
    colorscale = [[v, 'rgb' + str(color)] for v, color in zip(values, colors)]
    colorbar = {"tick0": 0, "tickmode": "array", "tickvals": tickvalues}

    return colorscale, colorbar


def _plotly_hist_dict(title, xtitle, ytitle, mode='group', data=None, comment=None, layout_config=None):
    """
    Create a basic Plotly chart dictionary
    :param title: Chart title
    :param xtitle: X-Series title
    :param ytitle: Y-Series title
    :param mode: multiple histograms mode. optionals stack / group / relative. Default is 'group'.
    :param data: Data items
    :type data: list
    :param layout_config: dict
    :return: Plotly chart dict.
    """
    assert mode in ('stack', 'group', 'relative')

    plotly_object = {
        "data": data or [],
        "layout": {
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "xaxis": {
                "title": xtitle
            },
            "yaxis": {
                "title": ytitle
            },
            "barmode": mode,
            "bargap": 0.08,
            "bargroupgap": 0
        }
    }
    if layout_config:
        plotly_object["layout"] = merge_dicts(plotly_object["layout"], layout_config)

    return plotly_object


def _np_row_to_plotly_data_item(np_row, label, xlabels=None):
    """
    Convert a numpy data row into a Plotly chart data item
    :param np_row: numpy 1D data row
    :param label: Item label
    :return: A plotly data item dict.
    """
    bins = list(range(np_row.shape[0])) if xlabels is None else list(xlabels)
    # mylabels = ['"' + label + '"'] * len(bins)
    this_trace_data = {
        "name": label,
        "y": np_row.tolist(),
        "x": bins,
        # "text": mylabels,
        "type": "bar"
    }
    return this_trace_data


def _plotly_scatter_layout_dict(title="Scatter", xaxis_title="X", yaxis_title="Y", series=None, comment=None):
    return {
        "data": [],
        "layout": {
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "xaxis": {
                "title": xaxis_title,
                "showspikes": True,
                "spikethickness": 1,
                "spikesnap": "cursor",
                "spikemode": "toaxis+across",
            },
            "yaxis": {
                "title": yaxis_title,
                "showspikes": True,
                "spikethickness": 1,
                "spikesnap": "cursor",
                "spikemode": "toaxis+across",
            },
            "name": series,
        }
    }


def plotly_scatter3d_layout_dict(title="Scatter", xaxis_title="X", yaxis_title="Y", zaxis_title="Z",
                                 series=None, show_legend=True, comment=None, layout_config=None):
    plotly_object = {
        "data": [],
        "layout": {
            "showlegend": show_legend,
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "scene": {
                'xaxis': {'title': xaxis_title},
                'yaxis': {'title': yaxis_title},
                'zaxis': {'title': zaxis_title},
            },
            "name": series,
        }
    }

    if layout_config:
        plotly_object["layout"] = merge_dicts(plotly_object["layout"], layout_config)

    return plotly_object


def create_plotly_table(table_plot, title, series, layout_config=None):
    """
    Create a basic Plotly table json style to be sent

    :param table_plot: the output table in pandas.DataFrame structure or list of rows (list) in a table
    :param title: Title (AKA metric)
    :type title: str
    :param series: Series (AKA variant)
    :type series: str
    :param layout_config: additional configuration layout
    :return: dict with plotly data.
    """
    is_list = isinstance(table_plot, (list, tuple))
    if is_list and (not table_plot or not any(table_plot)):
        # The list if empty
        headers_values = []
        cells_values = []
    elif is_list and table_plot[0] and isinstance(table_plot[0], (list, tuple)):
        headers_values = table_plot[0]
        cells_values = [list(i) for i in zip(*table_plot[1:])]
    else:
        if not pd:
            raise UsageError(
                "pandas is required in order to support reporting tables using CSV or a URL, "
                "please install the pandas python package"
            )
        index_added = not isinstance(table_plot.index, pd.RangeIndex)
        headers_values = list([col] for col in table_plot.columns)
        cells_values = json.loads(table_plot.T.to_json(orient='values', date_format='iso'))
        if index_added:
            if isinstance(table_plot.index, pd.MultiIndex):
                headers_values = [n or "" for n in (table_plot.index.names or [])] + headers_values
                cells_values = list(zip(*(table_plot.index.values.tolist() or []))) + cells_values
            else:
                headers_values.insert(0, table_plot.index.name or "")
                cells_values.insert(0, table_plot.index.values.tolist())

    ret = {
        "data": [{
            'type': 'table',
            'header': {
                'values': headers_values,
                'align': "left",
                'line': {'width': 0.5, 'color': '#d4d6e0'},
                'fill': {'color': "#fff"},
                'font': {'family': "Heebo, verdana, arial, sans-serif", 'size': 12, 'color': "#333"}
            },
            'cells': {
                'height': 30,
                'values': cells_values,
                'align': "left",
                'line': {'color': "white", 'width': 1},
                'font': {'family': "Heebo, verdana, arial, sans-serif", 'size': 14, 'color': "#384161"},
            }
        }],
        "layout": {
            "title": title,
            "name": series,
        }
    }
    if layout_config:
        ret["layout"] = merge_dicts(ret["layout"], layout_config)

    return ret


# clearml/utilities/proxy_object.py

import itertools
from copy import copy

import six


class ProxyDictPostWrite(dict):
    """ Dictionary wrapper that updates an arguments instance on any item set in the dictionary """

    def __init__(self, update_obj, update_func, *args, **kwargs):
        super(ProxyDictPostWrite, self).__init__(*args, **kwargs)
        self._update_obj = update_obj
        self._update_func = None
        for k, i in self.items():
            if isinstance(i, dict):
                super(ProxyDictPostWrite, self).update({k: ProxyDictPostWrite(update_obj, self._set_callback, i)})
        self._update_func = update_func

    def __setitem__(self, key, value):
        super(ProxyDictPostWrite, self).__setitem__(key, value)
        self._set_callback()

    def __reduce__(self):
        return dict, (), None, None, iter(self._to_dict().items())

    def _set_callback(self, *_):
        if self._update_func:
            self._update_func(self._update_obj, self)

    def _to_dict(self):
        a_dict = {}
        for k, i in self.items():
            if isinstance(i, ProxyDictPostWrite):
                a_dict[k] = i._to_dict()
            else:
                a_dict[k] = i
        return a_dict

    def update(self, E=None, **F):
        return super(ProxyDictPostWrite, self).update(
            ProxyDictPostWrite(self._update_obj, self._set_callback, E) if E is not None else
            ProxyDictPostWrite(self._update_obj, self._set_callback, **F))


class ProxyDictPreWrite(dict):
    """ Dictionary wrapper that prevents modifications to the dictionary """

    def __init__(self, update_obj, update_func, *args, **kwargs):
        super(ProxyDictPreWrite, self).__init__(*args, **kwargs)
        self._update_func = None
        for k, i in self.items():
            if isinstance(i, dict):
                self.update({k: ProxyDictPreWrite(k, self._nested_callback, i)})
        self._update_obj = update_obj
        self._update_func = update_func

    def __reduce__(self):
        return dict, (), None, None, iter(self.items())

    def __setitem__(self, key, value):
        key_value = self._set_callback((key, value,))
        if key_value:
            super(ProxyDictPreWrite, self).__setitem__(*key_value)

    def _set_callback(self, key_value, *_):
        if self._update_func is not None:
            if callable(self._update_func):
                res = self._update_func(self._update_obj, key_value)
            else:
                res = self._update_func

            if not res:
                return None
            return res
        return key_value

    def _nested_callback(self, prefix, key_value):
        return self._set_callback((prefix + '.' + key_value[0], key_value[1],))


def verify_basic_type(a_dict_list, basic_types=None):
    basic_types = (float, int, bool, six.string_types, ) if not basic_types else \
        tuple(b for b in basic_types if b not in (list, tuple, dict))

    if isinstance(a_dict_list, basic_types):
        return True
    if isinstance(a_dict_list, (list, tuple)):
        return all(verify_basic_type(v) for v in a_dict_list)
    elif isinstance(a_dict_list, dict):
        return all(verify_basic_type(k) for k in a_dict_list.keys()) and \
               all(verify_basic_type(v) for v in a_dict_list.values())


def flatten_dictionary(a_dict, prefix='', sep='/'):
    flat_dict = {}
    basic_types = (float, int, bool, six.string_types, )
    for k, v in a_dict.items():
        k = str(k)
        if isinstance(v, (float, int, bool, six.string_types)):
            flat_dict[prefix + k] = v
        elif isinstance(v, (list, tuple)) and all([isinstance(i, basic_types) for i in v]):
            flat_dict[prefix + k] = v
        elif isinstance(v, dict):
            nested_flat_dict = flatten_dictionary(v, prefix=prefix + k + sep, sep=sep)
            if nested_flat_dict:
                flat_dict.update(nested_flat_dict)
            else:
                flat_dict[k] = {}
        else:
            # this is a mixture of list and dict, or any other object,
            # leave it as is, we have nothing to do with it.
            flat_dict[prefix + k] = v
    return flat_dict


def nested_from_flat_dictionary(a_dict, flat_dict, prefix='', sep='/'):
    basic_types = (float, int, bool, six.string_types, )
    org_dict = copy(a_dict)
    for k, v in org_dict.items():
        k = str(k)
        if isinstance(v, (float, int, bool, six.string_types)):
            a_dict[k] = flat_dict.get(prefix + k, v)
        elif isinstance(v, (list, tuple)) and all([isinstance(i, basic_types) for i in v]):
            a_dict[k] = flat_dict.get(prefix + k, v)
        elif isinstance(v, dict):
            a_dict[k] = nested_from_flat_dictionary(v, flat_dict, prefix=prefix + k + sep, sep=sep) or v
        else:
            # this is a mixture of list and dict, or any other object,
            # leave it as is, we have nothing to do with it.
            a_dict[k] = flat_dict.get(prefix + k, v)
    return a_dict


def naive_nested_from_flat_dictionary(flat_dict, sep='/'):
    """ A naive conversion of a flat dictionary with '/'-separated keys signifying nesting
    into a nested dictionary.
    """
    return {
        sub_prefix: (
            bucket[0][1] if (len(bucket) == 1 and sub_prefix == bucket[0][0])
            else naive_nested_from_flat_dictionary(
                {
                    k[len(sub_prefix) + 1:]: v
                    for k, v in bucket
                    if len(k) > len(sub_prefix)
                }, sep=sep
            )
        )
        for sub_prefix, bucket in (
            (key, list(group))
            for key, group in itertools.groupby(
                sorted(flat_dict.items()),
                key=lambda item: item[0].partition(sep)[0]
            )
        )
    }


class WrapperBase(type):

    # This metaclass is heavily inspired by the Object Proxying python recipe
    # (http://code.activestate.com/recipes/496741/). It adds special methods
    # to the wrapper class so it can proxy the wrapped class. In addition, it
    # adds a field __overrides__ in the wrapper class dictionary, containing
    # all attributes decorated to be overriden.

    _special_names = [
        '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
        '__contains__', '__delitem__', '__delslice__', '__div__', '__divmod__',
        '__eq__', '__float__', '__floordiv__', '__ge__', '__getitem__',
        '__getslice__', '__gt__', '__hash__', '__hex__', '__iadd__', '__iand__',
        '__idiv__', '__idivmod__', '__ifloordiv__', '__ilshift__', '__imod__',
        '__imul__', '__int__', '__invert__', '__ior__', '__ipow__', '__irshift__',
        '__isub__', '__iter__', '__itruediv__', '__ixor__', '__le__', '__len__',
        '__long__', '__lshift__', '__lt__', '__mod__', '__mul__', '__ne__',
        '__neg__', '__oct__', '__or__', '__pos__', '__pow__', '__radd__',
        '__rand__', '__rdiv__', '__rdivmod__', '__reduce__', '__reduce_ex__',
        '__repr__', '__reversed__', '__rfloorfiv__', '__rlshift__', '__rmod__',
        '__rmul__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
        '__rtruediv__', '__rxor__', '__setitem__', '__setslice__', '__sub__',
        '__truediv__', '__xor__', 'next', '__str__', '__repr__',
    ]

    def __new__(mcs, classname, bases, attrs):
        def make_method(name):
            def method(self, *args, **kwargs):
                obj = object.__getattribute__(self, "_wrapped")
                if obj is None:
                    cb = object.__getattribute__(self, "_callback")
                    obj = cb()
                    object.__setattr__(self, '_wrapped', obj)

                # we have to convert the instance to the real type
                if args and len(args) == 1 and (
                        type(args[0]) == LazyEvalWrapper or hasattr(type(args[0]), '_base_class_')):
                    try:
                        int(args[0])  # force loading the instance
                    except:  # noqa
                        pass
                    args = (object.__getattribute__(args[0], "_wrapped"), )

                mtd = getattr(obj, name)
                return mtd(*args, **kwargs)
            return method

        typed_class = attrs.get('_base_class_')
        for name in mcs._special_names:
            if not typed_class or hasattr(typed_class, name):
                attrs[name] = make_method(name)

        overrides = attrs.get('__overrides__', [])
        # overrides.extend(k for k, v in attrs.items() if isinstance(v, lazy))
        attrs['__overrides__'] = overrides
        return type.__new__(mcs, classname, bases, attrs)


class LazyEvalWrapper(six.with_metaclass(WrapperBase)):

    # This class acts as a proxy for the wrapped instance it is passed. All
    # access to its attributes are delegated to the wrapped class, except
    # those contained in __overrides__.

    __slots__ = ['_wrapped', '_callback', '_remote_reference', '__weakref__']

    _remote_reference_calls = []

    def __init__(self, callback, remote_reference=None):
        object.__setattr__(self, '_wrapped', None)
        object.__setattr__(self, '_callback', callback)
        object.__setattr__(self, '_remote_reference', remote_reference)
        if remote_reference:
            LazyEvalWrapper._remote_reference_calls.append(remote_reference)

    def _remoteref(self):
        func = object.__getattribute__(self, "_remote_reference")
        if func and func in LazyEvalWrapper._remote_reference_calls:
            LazyEvalWrapper._remote_reference_calls.remove(func)

        return func() if callable(func) else func

    def __getattribute__(self, attr):
        if attr in ('__isabstractmethod__', ):
            return None
        if attr in ('_remoteref', '_remote_reference'):
            return object.__getattribute__(self, attr)
        return getattr(LazyEvalWrapper._load_object(self), attr)

    def __setattr__(self, attr, value):
        setattr(LazyEvalWrapper._load_object(self), attr, value)

    def __delattr__(self, attr):
        delattr(LazyEvalWrapper._load_object(self), attr)

    def __nonzero__(self):
        return bool(LazyEvalWrapper._load_object(self))

    def __bool__(self):
        return bool(LazyEvalWrapper._load_object(self))

    @staticmethod
    def _load_object(self):
        obj = object.__getattribute__(self, "_wrapped")
        if obj is None:
            cb = object.__getattribute__(self, "_callback")
            obj = cb()
            object.__setattr__(self, '_wrapped', obj)
        return obj

    @classmethod
    def trigger_all_remote_references(cls):
        for func in cls._remote_reference_calls:
            if callable(func):
                func()
        cls._remote_reference_calls = []


def lazy_eval_wrapper_spec_class(class_type):
    class TypedLazyEvalWrapper(six.with_metaclass(WrapperBase)):
        _base_class_ = class_type
        __slots__ = ['_wrapped', '_callback', '__weakref__']

        def __init__(self, callback):
            object.__setattr__(self, '_wrapped', None)
            object.__setattr__(self, '_callback', callback)

        def __nonzero__(self):
            return bool(LazyEvalWrapper._load_object(self))

        def __bool__(self):
            return bool(LazyEvalWrapper._load_object(self))

        def __getattribute__(self, attr):
            if attr == '__isabstractmethod__':
                return None
            if attr == '__class__':
                return class_type

            return getattr(LazyEvalWrapper._load_object(self), attr)

    return TypedLazyEvalWrapper


# clearml/utilities/py3_interop.py

""" Convenience classes supporting python3-like concepts """
import abc

import six


@six.add_metaclass(abc.ABCMeta)
class AbstractContextManager(object):
    """An abstract base class for context managers. Supported in contextlib from python 3.6 and up """

    def __enter__(self):
        """Return `self` upon entering the runtime context."""
        return self

    @abc.abstractmethod
    def __exit__(self, exc_type, exc_value, traceback):
        """Raise any exception triggered within the runtime context."""
        return None

    @classmethod
    def __subclasshook__(cls, C):
        if cls is AbstractContextManager:
            if (any("__enter__" in B.__dict__ for B in C.__mro__) and any("__exit__" in B.__dict__ for B in C.__mro__)):
                return True
        return NotImplemented


try:

    from abc import abstractclassmethod

except ImportError:

    class abstractclassmethod(classmethod):
        __isabstractmethod__ = True

        def __init__(self, callable):
            callable.__isabstractmethod__ = True
            super(abstractclassmethod, self).__init__(callable)


# clearml/utilities/resource_monitor.py

import logging
import os
import platform
import sys
import warnings
from time import time

import psutil
from pathlib2 import Path
from typing import Text

from .process.mp import BackgroundMonitor
from ..backend_api import Session
from ..binding.frameworks.tensorflow_bind import IsTensorboardInit

try:
    from .gpu import gpustat
except ImportError:
    gpustat = None


class ResourceMonitor(BackgroundMonitor):
    _title_machine = ':monitor:machine'
    _title_gpu = ':monitor:gpu'

    def __init__(self, task, sample_frequency_per_sec=2., report_frequency_sec=30.,
                 first_report_sec=None, wait_for_first_iteration_to_start_sec=180.0,
                 max_wait_for_first_iteration_to_start_sec=1800., report_mem_used_per_process=True):
        super(ResourceMonitor, self).__init__(task=task, wait_period=sample_frequency_per_sec)
        self._task = task
        self._sample_frequency = sample_frequency_per_sec
        self._report_frequency = report_frequency_sec
        self._first_report_sec = first_report_sec or report_frequency_sec
        self.wait_for_first_iteration = wait_for_first_iteration_to_start_sec
        self.max_check_first_iteration = max_wait_for_first_iteration_to_start_sec
        self._num_readouts = 0
        self._readouts = {}
        self._previous_readouts = {}
        self._previous_readouts_ts = time()
        self._gpustat_fail = 0
        self._gpustat = gpustat
        self._active_gpus = None
        self._process_info = psutil.Process() if report_mem_used_per_process else None
        self._last_process_pool = {}
        self._last_process_id_list = []
        if not self._gpustat:
            self._task.get_logger().report_text('ClearML Monitor: GPU monitoring is not available')
        else:  # if running_remotely():
            # noinspection PyBroadException
            try:
                active_gpus = os.environ.get('NVIDIA_VISIBLE_DEVICES', '') or \
                    os.environ.get('CUDA_VISIBLE_DEVICES', '')
                if active_gpus:
                    self._active_gpus = [int(g.strip()) for g in active_gpus.split(',')]
            except Exception:
                pass

    def daemon(self):
        if self._is_thread_mode_and_not_main_process():
            return

        seconds_since_started = 0
        reported = 0
        last_iteration = 0
        fallback_to_sec_as_iterations = None

        # get max GPU ID, and make sure our active list is within range
        if self._active_gpus:
            # noinspection PyBroadException
            try:
                gpu_stat = self._gpustat.new_query()
                if max(self._active_gpus) > len(gpu_stat.gpus) - 1:
                    self._active_gpus = None
            except Exception:
                pass

        # add Task runtime_properties with the machine spec
        if Session.check_min_api_version('2.13'):
            try:
                machine_spec = self._get_machine_specs()
                if machine_spec:
                    # noinspection PyProtectedMember
                    self._task._set_runtime_properties(runtime_properties=machine_spec)
            except Exception as ex:
                logging.getLogger('clearml.resource_monitor').debug(
                    'Failed logging machine specification: {}'.format(ex))

        # last_iteration_interval = None
        # last_iteration_ts = 0
        # repeated_iterations = 0
        while True:
            last_report = time()
            current_report_frequency = self._report_frequency if reported != 0 else self._first_report_sec
            while (time() - last_report) < current_report_frequency:
                # wait for self._sample_frequency seconds, if event set quit
                if self._event.wait(1.0 / self._sample_frequency):
                    return
                # noinspection PyBroadException
                try:
                    self._update_readouts()
                except Exception:
                    pass

            seconds_since_started += int(round(time() - last_report))
            # check if we do not report any metric (so it means the last iteration will not be changed)
            if fallback_to_sec_as_iterations is None:
                if IsTensorboardInit.tensorboard_used():
                    fallback_to_sec_as_iterations = False
                elif seconds_since_started >= self.wait_for_first_iteration:
                    self._task.get_logger().report_text('ClearML Monitor: Could not detect iteration reporting, '
                                                        'falling back to iterations as seconds-from-start')
                    fallback_to_sec_as_iterations = True
            elif fallback_to_sec_as_iterations is True and seconds_since_started <= self.max_check_first_iteration:
                if self._check_logger_reported():
                    fallback_to_sec_as_iterations = False
                    self._task.get_logger().report_text('ClearML Monitor: Reporting detected, '
                                                        'reverting back to iteration based reporting')

            clear_readouts = True
            # if we do not have last_iteration, we just use seconds as iteration
            if fallback_to_sec_as_iterations:
                iteration = seconds_since_started
            else:
                iteration = self._task.get_last_iteration()
                if iteration < last_iteration:
                    # we started a new session?!
                    # wait out
                    clear_readouts = False
                    iteration = last_iteration
                elif iteration == last_iteration:
                    # repeated_iterations += 1
                    # if last_iteration_interval:
                    #     # to be on the safe side, we don't want to pass the actual next iteration
                    #     iteration += int(0.95*last_iteration_interval[0] * (seconds_since_started - last_iteration_ts)
                    #                      / last_iteration_interval[1])
                    # else:
                    #     iteration += 1
                    clear_readouts = False
                    iteration = last_iteration
                else:
                    # last_iteration_interval = (iteration - last_iteration, seconds_since_started - last_iteration_ts)
                    # repeated_iterations = 0
                    # last_iteration_ts = seconds_since_started
                    last_iteration = iteration
                    fallback_to_sec_as_iterations = False
                    clear_readouts = True

            # start reporting only when we figured out, if this is seconds based, or iterations based
            average_readouts = self._get_average_readouts()
            if fallback_to_sec_as_iterations is not None:
                for k, v in average_readouts.items():
                    # noinspection PyBroadException
                    try:
                        title = self._title_gpu if k.startswith('gpu_') else self._title_machine
                        # 3 points after the dot
                        value = round(v * 1000) / 1000.
                        self._task.get_logger().report_scalar(title=title, series=k, iteration=iteration, value=value)
                    except Exception:
                        pass
                # clear readouts if this is update is not averaged
                if clear_readouts:
                    self._clear_readouts()

            # count reported iterations
            reported += 1

    def _update_readouts(self):
        readouts = self._machine_stats()
        elapsed = time() - self._previous_readouts_ts
        self._previous_readouts_ts = time()
        for k, v in readouts.items():
            # cumulative measurements
            if k.endswith('_mbs'):
                v = (v - self._previous_readouts.get(k, v)) / elapsed

            self._readouts[k] = self._readouts.get(k, 0.0) + v
        self._num_readouts += 1
        self._previous_readouts = readouts

    def _get_num_readouts(self):
        return self._num_readouts

    def _get_average_readouts(self):
        average_readouts = dict((k, v / float(self._num_readouts)) for k, v in self._readouts.items())
        return average_readouts

    def _clear_readouts(self):
        self._readouts = {}
        self._num_readouts = 0

    def _machine_stats(self):
        """
        :return: machine stats dictionary, all values expressed in megabytes
        """
        cpu_usage = [float(v) for v in psutil.cpu_percent(percpu=True)]
        stats = {
            "cpu_usage": sum(cpu_usage) / float(len(cpu_usage)),
        }

        bytes_per_megabyte = 1024 ** 2

        def bytes_to_megabytes(x):
            return x / bytes_per_megabyte

        virtual_memory = psutil.virtual_memory()
        # stats["memory_used_gb"] = bytes_to_megabytes(virtual_memory.used) / 1024
        stats["memory_used_gb"] = bytes_to_megabytes(
            self._get_process_used_memory() if self._process_info else virtual_memory.used) / 1024
        stats["memory_free_gb"] = bytes_to_megabytes(virtual_memory.available) / 1024
        disk_use_percentage = psutil.disk_usage(Text(Path.home())).percent
        stats["disk_free_percent"] = 100.0 - disk_use_percentage
        with warnings.catch_warnings():
            if logging.root.level > logging.DEBUG:  # If the logging level is bigger than debug, ignore
                # psutil.sensors_temperatures warnings
                warnings.simplefilter("ignore", category=RuntimeWarning)
            sensor_stat = (psutil.sensors_temperatures() if hasattr(psutil, "sensors_temperatures") else {})
        if "coretemp" in sensor_stat and len(sensor_stat["coretemp"]):
            stats["cpu_temperature"] = max([float(t.current) for t in sensor_stat["coretemp"]])

        # protect against permission issues
        # update cached measurements
        # noinspection PyBroadException
        try:
            net_stats = psutil.net_io_counters()
            stats["network_tx_mbs"] = bytes_to_megabytes(net_stats.bytes_sent)
            stats["network_rx_mbs"] = bytes_to_megabytes(net_stats.bytes_recv)
        except Exception:
            pass

        # protect against permission issues
        # noinspection PyBroadException
        try:
            io_stats = psutil.disk_io_counters()
            stats["io_read_mbs"] = bytes_to_megabytes(io_stats.read_bytes)
            stats["io_write_mbs"] = bytes_to_megabytes(io_stats.write_bytes)
        except Exception:
            pass

        # check if we can access the gpu statistics
        if self._gpustat:
            # noinspection PyBroadException
            try:
                stats.update(self._get_gpu_stats())
            except Exception:
                # something happened and we can't use gpu stats,
                self._gpustat_fail += 1
                if self._gpustat_fail >= 3:
                    self._task.get_logger().report_text('ClearML Monitor: GPU monitoring failed getting GPU reading, '
                                                        'switching off GPU monitoring')
                    self._gpustat = None

        return stats

    def _check_logger_reported(self):
        titles = self.get_logger_reported_titles(self._task)
        return len(titles) > 0

    @classmethod
    def get_logger_reported_titles(cls, task):
        # noinspection PyProtectedMember
        titles = list(task.get_logger()._get_used_title_series().keys())
        try:
            titles.remove(cls._title_machine)
        except ValueError:
            pass
        try:
            titles.remove(cls._title_gpu)
        except ValueError:
            pass
        return titles

    def _get_process_used_memory(self):
        def mem_usage_children(a_mem_size, pr, parent_mem=None):
            self._last_process_id_list.append(pr.pid)
            # add out memory usage
            our_mem = pr.memory_info()
            mem_diff = our_mem.rss - parent_mem.rss if parent_mem else our_mem.rss
            a_mem_size += mem_diff if mem_diff > 0 else 0
            # now we are the parent
            for child in pr.children():
                # get the current memory
                m = pr.memory_info()
                mem_diff = m.rss - our_mem.rss
                a_mem_size += mem_diff if mem_diff > 0 else 0
                a_mem_size = mem_usage_children(a_mem_size, child, parent_mem=m)
            return a_mem_size

        # only run the memory usage query once per reporting period
        # because this memory query is relatively slow, and changes very little.
        if self._last_process_pool.get('cpu') and \
                (time() - self._last_process_pool['cpu'][0]) < self._report_frequency:
            return self._last_process_pool['cpu'][1]

        # if we have no parent process, return 0 (it's an error)
        if not self._process_info:
            return 0

        self._last_process_id_list = []
        mem_size = mem_usage_children(0, self._process_info)
        self._last_process_pool['cpu'] = time(), mem_size

        return mem_size

    def _get_gpu_stats(self):
        if not self._gpustat:
            return {}

        # per process memory query id slow, so we only call it once per reporting period,
        # On the rest of the samples we return the previous memory measurement

        # update mem used by our process and sub processes
        if self._process_info and (not self._last_process_pool.get('gpu') or
                                   (time() - self._last_process_pool['gpu'][0]) >= self._report_frequency):
            gpu_stat = self._gpustat.new_query(per_process_stats=True)
            gpu_mem = {}
            for i, g in enumerate(gpu_stat.gpus):
                # only monitor the active gpu's, if none were selected, monitor everything
                if self._active_gpus and i not in self._active_gpus:
                    continue
                gpu_mem[i] = 0
                for p in g.processes:
                    if p['pid'] in self._last_process_id_list:
                        gpu_mem[i] += p.get('gpu_memory_usage', 0)
            self._last_process_pool['gpu'] = time(), gpu_mem
        else:
            # if we do no need to update the memory usage, run global query
            # if we have no parent process (backward compatibility), return global stats
            gpu_stat = self._gpustat.new_query()
            gpu_mem = self._last_process_pool['gpu'][1] if self._last_process_pool.get('gpu') else None

        # generate the statistics dict for actual report
        stats = {}
        for i, g in enumerate(gpu_stat.gpus):
            # only monitor the active gpu's, if none were selected, monitor everything
            if self._active_gpus and i not in self._active_gpus:
                continue
            stats["gpu_%d_temperature" % i] = float(g["temperature.gpu"])
            stats["gpu_%d_utilization" % i] = float(g["utilization.gpu"])
            stats["gpu_%d_mem_usage" % i] = 100. * float(g["memory.used"]) / float(g["memory.total"])
            # already in MBs
            stats["gpu_%d_mem_free_gb" % i] = float(g["memory.total"] - g["memory.used"]) / 1024
            # use previously sampled process gpu memory, or global if it does not exist
            stats["gpu_%d_mem_used_gb" % i] = float(gpu_mem[i] if gpu_mem else g["memory.used"]) / 1024

        return stats

    def _get_machine_specs(self):
        # type: () -> dict
        specs = {}
        # noinspection PyBroadException
        try:
            specs = {
                'platform': str(sys.platform),
                'python_version': str(platform.python_version()),
                'python_exec': str(sys.executable),
                'OS': str(platform.platform(aliased=True)),
                'processor': str(platform.machine()),
                'cpu_cores': int(psutil.cpu_count()),
                'memory_gb': round(psutil.virtual_memory().total / 1024 ** 3, 1),
                'hostname': str(platform.node()),
                'gpu_count': 0,
            }
            if self._gpustat:
                gpu_stat = self._gpustat.new_query(shutdown=True, get_driver_info=True)
                if gpu_stat.gpus:
                    gpus = [g for i, g in enumerate(gpu_stat.gpus) if not self._active_gpus or i in self._active_gpus]
                    specs.update(
                        gpu_count=int(len(gpus)),
                        gpu_type=', '.join(g.name for g in gpus),
                        gpu_memory=', '.join('{}GB'.format(round(g.memory_total/1024.0)) for g in gpus),
                        gpu_driver_version=gpu_stat.driver_version or '',
                        gpu_driver_cuda_version=gpu_stat.driver_cuda_version or '',
                    )

        except Exception:
            pass

        return specs


# clearml/utilities/seed.py

import sys
import random

try:
    import numpy as np
except Exception:
    np = None


def make_deterministic(seed=1337, cudnn_deterministic=False):
    """
    Ensure deterministic behavior across PyTorch using the provided random seed.
    This function makes sure that torch, numpy and random use the same random seed.

    When using clearml's task, call this function using the task's random seed like so:
        make_deterministic(task.get_random_seed())

    :param int seed: Seed number
    :param bool cudnn_deterministic: In order to make computations deterministic on your specific platform
    and PyTorch release, set this value to True. torch will only allow those CuDNN algorithms that are
    (believed to be) deterministic. This can have a performance impact (slower execution) depending on your model.
    """
    seed = int(seed) & 0xFFFFFFFF
    torch = sys.modules.get("torch")
    tf = sys.modules.get("tensorflow")

    if cudnn_deterministic:
        try:
            torch.backends.cudnn.deterministic = True
            torch.backends.cudnn.benchmark = False
        except Exception:
            pass

    random.seed(seed)

    if np is not None:
        np.random.seed(seed)

    if torch is not None:
        try:
            torch.manual_seed(seed)
            torch.cuda.manual_seed(seed)
        except Exception:
            pass

    if tf is not None:
        # reset graph state
        try:
            import tensorflow  # noqa: F401
            from tensorflow.python.eager.context import _context
            eager_mode_bypass = _context is None
        except Exception:
            eager_mode_bypass = False

        if not eager_mode_bypass:
            try:
                tf.compat.v1.set_random_seed(seed)
            except Exception:
                try:
                    tf.set_random_seed(seed)
                except Exception:
                    pass

            try:
                tf.compat.v1.random.set_random_seed(seed)
            except Exception:
                try:
                    tf.random.set_random_seed(seed)
                except Exception:
                    pass


# clearml/utilities/version.py

from __future__ import absolute_import, division, print_function

import collections
import re

import six

if six.PY3:
    from math import inf
else:
    inf = float('inf')


class InvalidVersion(ValueError):
    """
    An invalid version was found, users should refer to PEP 440.
    """


_Version = collections.namedtuple(
    "_Version", ["epoch", "release", "dev", "pre", "post", "local"]
)


class _BaseVersion(object):
    def __init__(self, key):
        self._key = key

    def __hash__(self):
        return hash(self._key)

    def __lt__(self, other):
        return self._compare(other, lambda s, o: s < o)

    def __le__(self, other):
        return self._compare(other, lambda s, o: s <= o)

    def __eq__(self, other):
        return self._compare(other, lambda s, o: s == o)

    def __ge__(self, other):
        return self._compare(other, lambda s, o: s >= o)

    def __gt__(self, other):
        return self._compare(other, lambda s, o: s > o)

    def __ne__(self, other):
        return self._compare(other, lambda s, o: s != o)

    def _compare(self, other, method):
        if not isinstance(other, _BaseVersion):
            return NotImplemented

        return method(self._key, other._key)


class Version(_BaseVersion):
    VERSION_PATTERN = r"""
        v?
        (?:
            (?:(?P<epoch>[0-9]+)!)?                           # epoch
            (?P<release>[0-9]+(?:\.[0-9]+)*)                  # release segment
            (?P<pre>                                          # pre-release
                [-_\.]?
                (?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview))
                [-_\.]?
                (?P<pre_n>[0-9]+)?
            )?
            (?P<post>                                         # post release
                (?:-(?P<post_n1>[0-9]+))
                |
                (?:
                    [-_\.]?
                    (?P<post_l>post|rev|r)
                    [-_\.]?
                    (?P<post_n2>[0-9]+)?
                )
            )?
            (?P<dev>                                          # dev release
                [-_\.]?
                (?P<dev_l>dev)
                [-_\.]?
                (?P<dev_n>[0-9]+)?
            )?
        )
        (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
    """
    _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE)
    _local_version_separators = re.compile(r"[\._-]")

    def __init__(self, version):
        # Validate the version and parse it into pieces
        match = self._regex.search(version)
        if not match:
            raise InvalidVersion("Invalid version: '{0}'".format(version))

        # Store the parsed out pieces of the version
        self._version = _Version(
            epoch=int(match.group("epoch")) if match.group("epoch") else 0,
            release=tuple(int(i) for i in match.group("release").split(".")),
            pre=self._parse_letter_version(match.group("pre_l"), match.group("pre_n")),
            post=self._parse_letter_version(
                match.group("post_l") or '', match.group("post_n1") or match.group("post_n2") or ''
            ),
            dev=self._parse_letter_version(match.group("dev_l") or '', match.group("dev_n") or ''),
            local=self._parse_local_version(match.group("local") or ''),
        )

        # Generate a key which will be used for sorting
        key = self._cmpkey(
            self._version.epoch,
            self._version.release,
            self._version.pre,
            self._version.post,
            self._version.dev,
            self._version.local,
        )

        super(Version, self).__init__(key)

    def __repr__(self):
        return "<Version({0})>".format(repr(str(self)))

    def __str__(self):
        parts = []

        # Epoch
        if self.epoch != 0:
            parts.append("{0}!".format(self.epoch))

        # Release segment
        parts.append(".".join(str(x) for x in self.release))

        # Pre-release
        if self.pre is not None:
            parts.append("".join(str(x) for x in self.pre))

        # Post-release
        if self.post is not None:
            parts.append(".post{0}".format(self.post))

        # Development release
        if self.dev is not None:
            parts.append(".dev{0}".format(self.dev))

        # Local version segment
        if self.local is not None:
            parts.append("+{0}".format(self.local))

        return "".join(parts)

    @property
    def epoch(self):
        return self._version.epoch

    @property
    def release(self):
        return self._version.release

    @property
    def pre(self):
        return self._version.pre

    @property
    def post(self):
        return self._version.post[1] if self._version.post else None

    @property
    def dev(self):
        return self._version.dev[1] if self._version.dev else None

    @property
    def local(self):
        if self._version.local:
            return ".".join(str(x) for x in self._version.local)
        else:
            return None

    @property
    def public(self):
        return str(self).split("+", 1)[0]

    @property
    def base_version(self):
        parts = []

        # Epoch
        if self.epoch != 0:
            parts.append("{0}!".format(self.epoch))

        # Release segment
        parts.append(".".join(str(x) for x in self.release))

        return "".join(parts)

    @property
    def is_prerelease(self):
        return self.dev is not None or self.pre is not None

    @property
    def is_postrelease(self):
        return self.post is not None

    @property
    def is_devrelease(self):
        return self.dev is not None

    @staticmethod
    def _parse_letter_version(letter, number):
        if not letter and not number:
            return None
        if letter:
            # We consider there to be an implicit 0 in a pre-release if there is
            # not a numeral associated with it.
            if number is None:
                number = 0

            # We normalize any letters to their lower case form
            letter = letter.lower()

            # We consider some words to be alternate spellings of other words and
            # in those cases we want to normalize the spellings to our preferred
            # spelling.
            if letter == "alpha":
                letter = "a"
            elif letter == "beta":
                letter = "b"
            elif letter in ["c", "pre", "preview"]:
                letter = "rc"
            elif letter in ["rev", "r"]:
                letter = "post"

            return letter, int(number)
        if not letter and number:
            # We assume if we are given a number, but we are not given a letter
            # then this is using the implicit post release syntax (e.g. 1.0-1)
            letter = "post"

        return letter, int(number)

    @classmethod
    def _parse_local_version(cls, local):
        """
        Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve").
        """
        if local is not None:
            local = tuple(
                part.lower() if not part.isdigit() else int(part)
                for part in cls._local_version_separators.split(local)
            )
            if not local or not local[0]:
                return None
            return local
        return None

    @staticmethod
    def _cmpkey(epoch, release, pre, post, dev, local):
        # When we compare a release version, we want to compare it with all of the
        # trailing zeros removed. So we'll use a reverse the list, drop all the now
        # leading zeros until we come to something non zero, then take the rest
        # re-reverse it back into the correct order and make it a tuple and use
        # that for our sorting key.
        # release = tuple(
        #     reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release))))
        # )

        # Versions without a pre-release (except as noted above) should sort after
        # those with one.
        if not pre:
            pre = inf
        elif pre:
            pre = pre[1]

        # Versions without a post segment should sort before those with one.
        if not post:
            post = -inf
        else:
            post = post[1]

        # Versions without a development segment should sort after those with one.
        if not dev:
            dev = inf
        else:
            dev = dev[1]

        if not local:
            # Versions without a local segment should sort before those with one.
            local = inf
        else:
            # Versions with a local segment need that segment parsed to implement
            # the sorting rules in PEP440.
            # - Alpha numeric segments sort before numeric segments
            # - Alpha numeric segments sort lexicographically
            # - Numeric segments sort numerically
            # - Shorter versions sort before longer versions when the prefixes
            #   match exactly
            local = local[1]

        return epoch, release, pre, post, dev, local


# setup.py

"""
ClearML - Artificial Intelligence Version Control
https://github.com/allegroai/clearml
"""

import os.path
# Always prefer setuptools over distutils
from setuptools import setup, find_packages


def read_text(filepath):
    with open(filepath, "r", encoding="utf-8") as f:
        return f.read()


here = os.path.dirname(__file__)
# Get the long description from the README file
long_description = read_text(os.path.join(here, 'README.md'))


def read_version_string(version_file):
    for line in read_text(version_file).splitlines():
        if line.startswith('__version__'):
            delim = '"' if '"' in line else "'"
            return line.split(delim)[1]
    else:
        raise RuntimeError("Unable to find version string.")


version = read_version_string("clearml/version.py")

requirements = read_text(os.path.join(here, 'requirements.txt')).splitlines()

setup(
    name='clearml',
    version=version,
    description='ClearML - Auto-Magical Experiment Manager, Version Control, and MLOps for AI',
    long_description=long_description,
    long_description_content_type='text/markdown',
    # The project's main homepage.
    url='https://github.com/allegroai/clearml',
    author='Allegroai',
    author_email='clearml@allegro.ai',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'Operating System :: POSIX :: Linux',
        'Operating System :: MacOS :: MacOS X',
        'Operating System :: Microsoft',
        'Topic :: Scientific/Engineering :: Artificial Intelligence',
        'Topic :: Software Development',
        'Topic :: Software Development :: Version Control',
        'Topic :: System :: Logging',
        'Topic :: System :: Monitoring',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'License :: OSI Approved :: Apache Software License',
    ],
    keywords='clearml trains development machine deep learning version control machine-learning machinelearning '
             'deeplearning deep-learning experiment-manager experimentmanager',
    packages=find_packages(exclude=['contrib', 'docs', 'data', 'examples', 'tests']),
    install_requires=requirements,
    extras_require={
        's3': [
            'boto3>=1.9',
        ],
        'azure': [
            'azure-storage-blob>=2.0.1,<=2.1',
        ],
        'gs': [
            'google-cloud-storage>=1.13.2',
        ],
    },
    package_data={
        'clearml': ['config/default/*.conf', 'backend_api/config/default/*.conf']
    },
    include_package_data=True,
    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={
        'console_scripts': [
            'clearml-init = clearml.cli.config.__main__:main',
            'clearml-data = clearml.cli.data.__main__:main',
            'clearml-task = clearml.cli.task.__main__:main',
        ],
    },
)


# clearml/binding/frameworks/__init__.py

import os
import shutil
import sys
import threading
from enum import Enum
from random import randint
from tempfile import mkstemp
from typing import TYPE_CHECKING, Callable, Dict, Optional, Any, Union

import six
from pathlib2 import Path

from ...backend_interface.model import Model
from ...config import running_remotely
from ...debugging.log import get_logger
from ...model import InputModel, OutputModel

if TYPE_CHECKING:
    from ...task import Task

TrainsFrameworkAdapter = 'frameworks'
_recursion_guard = {}


def _patched_call(original_fn, patched_fn):
    def _inner_patch(*args, **kwargs):
        # noinspection PyProtectedMember,PyUnresolvedReferences
        ident = threading._get_ident() if six.PY2 else threading.get_ident()
        if ident in _recursion_guard:
            return original_fn(*args, **kwargs)
        _recursion_guard[ident] = 1
        ret = None
        try:
            ret = patched_fn(original_fn, *args, **kwargs)
        except Exception as ex:
            raise ex
        finally:
            try:
                _recursion_guard.pop(ident)
            except KeyError:
                pass
        return ret

    return _inner_patch


class _Empty(object):
    def __init__(self):
        self.trains_in_model = None


class WeightsFileHandler(object):
    # _model_out_store_lookup = {}
    # _model_in_store_lookup = {}
    _model_store_lookup_lock = threading.Lock()
    _model_pre_callbacks = {}
    _model_post_callbacks = {}

    class CallbackType(Enum):
        def __str__(self):
            return str(self.value)

        def __eq__(self, other):
            return str(self) == str(other)

        save = 'save'
        load = 'load'

    class ModelInfo(object):
        def __init__(self, model, upload_filename, local_model_path, local_model_id, framework, task):
            # type: (Optional[Model], Optional[str], str, str, str, Task) -> None
            """
            :param model: None, OutputModel or InputModel
            :param upload_filename: example 'filename.ext'
            :param local_model_path: example /local/copy/filename.random_number.ext'
            :param local_model_id: example /local/copy/filename.ext'
            :param framework: example 'PyTorch'
            :param task: Task object
            """
            self.model = model
            self.upload_filename = upload_filename
            self.local_model_path = local_model_path
            self.local_model_id = local_model_id
            self.framework = framework
            self.task = task

    @staticmethod
    def _add_callback(func, target):
        # type: (Callable, Dict[int, Callable]) -> int

        if func in target.values():
            return [k for k, v in target.items() if v == func][0]

        while True:
            h = randint(0, 1 << 31)
            if h not in target:
                break

        target[h] = func
        return h

    @staticmethod
    def _remove_callback(handle, target):
        # type: (int, Dict[int, Callable]) -> bool
        if handle in target:
            target.pop(handle, None)
            return True
        return False

    @classmethod
    def add_pre_callback(cls, callback_function):
        # type: (Callable[[Union[str, WeightsFileHandler.CallbackType], WeightsFileHandler.ModelInfo], Optional[WeightsFileHandler.ModelInfo]]) -> int  # noqa
        """
        Add a pre-save/load callback for weights files and return its handle. If the callback was already added,
         return the existing handle.

        Use this callback to modify the weights filename registered in the ClearML Server. In case ClearML is
         configured to upload the weights file, this will affect the uploaded filename as well.
         Callback returning None will disable the tracking of the current call Model save,
         it will not disable saving it to disk, just the logging/tracking/uploading.

        :param callback_function: A function accepting action type ("load" or "save"),
            callback_function('load' or 'save', WeightsFileHandler.ModelInfo) -> WeightsFileHandler.ModelInfo
        :return Callback handle
        """
        return cls._add_callback(callback_function, cls._model_pre_callbacks)

    @classmethod
    def add_post_callback(cls, callback_function):
        # type: (Callable[[Union[str, WeightsFileHandler.CallbackType], WeightsFileHandler.ModelInfo], WeightsFileHandler.ModelInfo]) -> int  # noqa
        """
        Add a post-save/load callback for weights files and return its handle.
        If the callback was already added, return the existing handle.

        :param callback_function: A function accepting action type ("load" or "save"),
            callback_function('load' or 'save', WeightsFileHandler.ModelInfo) -> WeightsFileHandler.ModelInfo
        :return Callback handle
        """
        return cls._add_callback(callback_function, cls._model_post_callbacks)

    @classmethod
    def remove_pre_callback(cls, handle):
        # type: (int) -> bool
        """
        Add a pre-save/load callback for weights files and return its handle.
        If the callback was already added, return the existing handle.

        :param handle: A callback handle returned from :meth:WeightsFileHandler.add_pre_callback
        :return True if callback removed, False otherwise
        """
        return cls._remove_callback(handle, cls._model_pre_callbacks)

    @classmethod
    def remove_post_callback(cls, handle):
        # type: (int) -> bool
        """
        Add a pre-save/load callback for weights files and return its handle.
        If the callback was already added, return the existing handle.

        :param handle: A callback handle returned from :meth:WeightsFileHandler.add_post_callback
        :return True if callback removed, False otherwise
        """
        return cls._remove_callback(handle, cls._model_post_callbacks)

    @staticmethod
    def restore_weights_file(model, filepath, framework, task):
        # type: (Optional[Any], Optional[str], Optional[str], Optional[Task]) -> str
        if task is None:
            return filepath

        try:
            local_model_path = os.path.abspath(filepath) if filepath else filepath
        except TypeError:
            # not a recognized type, we just return it back
            return filepath

        model_info = WeightsFileHandler.ModelInfo(
            model=None, upload_filename=None, local_model_path=local_model_path,
            local_model_id=filepath, framework=framework, task=task)
        # call pre model callback functions
        for cb in WeightsFileHandler._model_pre_callbacks.values():
            # noinspection PyBroadException
            try:
                model_info = cb(WeightsFileHandler.CallbackType.load, model_info)
            except Exception:
                pass

        # if callback forced us to leave they return None
        if model_info is None:
            # callback forced quit
            return filepath

        if not model_info.local_model_path:
            # get_logger(TrainsFrameworkAdapter).debug("Could not retrieve model file location, model is not logged")
            return filepath

        try:
            WeightsFileHandler._model_store_lookup_lock.acquire()

            # check if object already has InputModel
            if model_info.model:
                trains_in_model = model_info.model
            else:
                # # disable model reuse, let Model module try to find it for use
                trains_in_model, ref_model = None, None  # noqa: F841
                # trains_in_model, ref_model = WeightsFileHandler._model_in_store_lookup.get(
                #     id(model) if model is not None else None, (None, None))
                # # noinspection PyCallingNonCallable
                # if ref_model is not None and model != ref_model():
                #     # old id pop it - it was probably reused because the object is dead
                #     WeightsFileHandler._model_in_store_lookup.pop(id(model))
                #     trains_in_model, ref_model = None, None

            # check if object already has InputModel
            model_name_id = getattr(model, 'name', '') if model else ''
            # noinspection PyBroadException
            try:
                config_text = None
                config_dict = trains_in_model.config_dict if trains_in_model else None
            except Exception:
                config_dict = None
                # noinspection PyBroadException
                try:
                    config_text = trains_in_model.config_text if trains_in_model else None
                except Exception:
                    config_text = None

            if not trains_in_model:
                # check if we already have the model object:
                # noinspection PyProtectedMember
                model_id, model_uri = Model._local_model_to_id_uri.get(
                    model_info.local_model_id or model_info.local_model_path, (None, None))
                if model_id:
                    # noinspection PyBroadException
                    try:
                        trains_in_model = InputModel(model_id)
                    except Exception:
                        model_id = None

                # if we do not, we need to import the model
                if not model_id:
                    trains_in_model = InputModel.import_model(
                        weights_url=model_info.local_model_path,
                        config_dict=config_dict,
                        config_text=config_text,
                        name=task.name + (' ' + model_name_id) if model_name_id else '',
                        label_enumeration=task.get_labels_enumeration(),
                        framework=framework,
                        create_as_published=False,
                    )

            model_info.model = trains_in_model
            # call post model callback functions
            for cb in WeightsFileHandler._model_post_callbacks.values():
                # noinspection PyBroadException
                try:
                    model_info = cb(WeightsFileHandler.CallbackType.load, model_info)
                except Exception:
                    pass
            trains_in_model = model_info.model

            # # disable model reuse, let Model module try to find it for use
            # if model is not None:
            #     # noinspection PyBroadException
            #     try:
            #         ref_model = weakref.ref(model)
            #     except Exception:
            #         ref_model = None
            #     WeightsFileHandler._model_in_store_lookup[id(model)] = (trains_in_model, ref_model)

            task.connect(trains_in_model)
            # if we are running remotely we should deserialize the object
            # because someone might have changed the config_dict
            # Hack: disabled
            if False and running_remotely():
                # reload the model
                model_config = trains_in_model.config_dict
                # verify that this is the same model so we are not deserializing a different model
                if (config_dict and config_dict.get('config') and model_config and model_config.get('config') and
                    config_dict.get('config').get('name') == model_config.get('config').get('name')) or \
                        (not config_dict and not model_config):
                    filepath = trains_in_model.get_weights()
                    # update filepath to point to downloaded weights file
                    # actual model weights loading will be done outside the try/exception block

            # update back the internal Model lookup, and replace the local file with our file
            # noinspection PyProtectedMember
            Model._local_model_to_id_uri[model_info.local_model_id] = (
                trains_in_model.id, trains_in_model.url)

        except Exception as ex:
            get_logger(TrainsFrameworkAdapter).debug(str(ex))
        finally:
            WeightsFileHandler._model_store_lookup_lock.release()

        return filepath

    @staticmethod
    def create_output_model(
            model,  # type: Optional[Any]
            saved_path,  # type: Optional[str]
            framework,  # type: Optional[str]
            task,  # type: Optional[Task]
            singlefile=False,  # type: bool
            model_name=None,  # type: Optional[str]
            config_obj=None  # type: Optional[Union[str, dict]]
    ):
        # type: (...) -> str
        if task is None:
            return saved_path

        try:
            WeightsFileHandler._model_store_lookup_lock.acquire()

            # # disable model reuse, let Model module try to find it for use
            trains_out_model, ref_model = None, None  # noqa: F841
            # check if object already has InputModel
            # trains_out_model, ref_model = WeightsFileHandler._model_out_store_lookup.get(
            #     id(model) if model is not None else None, (None, None))
            # # notice ref_model() is not an error/typo this is a weakref object call
            # # noinspection PyCallingNonCallable
            # if ref_model is not None and model != ref_model():
            #     # old id pop it - it was probably reused because the object is dead
            #     WeightsFileHandler._model_out_store_lookup.pop(id(model))
            #     trains_out_model, ref_model = None, None

            try:
                local_model_path = os.path.abspath(saved_path) if saved_path else saved_path
            except TypeError:
                # not a recognized type:
                return saved_path

            model_info = WeightsFileHandler.ModelInfo(
                model=trains_out_model, upload_filename=None, local_model_path=local_model_path,
                local_model_id=saved_path, framework=framework, task=task)

            if not model_info.local_model_path:
                # get_logger(TrainsFrameworkAdapter).debug(
                #     "Could not retrieve model location, skipping auto model logging")
                return saved_path

            # check if we have output storage, and generate list of files to upload
            if Path(model_info.local_model_path).is_dir():
                files = [str(f) for f in Path(model_info.local_model_path).rglob('*')]
            elif singlefile:
                files = [str(Path(model_info.local_model_path).absolute())]
            else:
                files = [str(f) for f in Path(model_info.local_model_path).parent.glob(
                    str(Path(model_info.local_model_path).name) + '.*')]

            target_filename = None
            if len(files) > 1:
                # noinspection PyBroadException
                try:
                    target_filename = Path(model_info.local_model_path).stem
                except Exception:
                    pass
            else:
                target_filename = Path(files[0]).name

            # call pre model callback functions
            model_info.upload_filename = target_filename
            for cb in WeightsFileHandler._model_pre_callbacks.values():
                # noinspection PyBroadException
                try:
                    model_info = cb(WeightsFileHandler.CallbackType.save, model_info)
                except Exception:
                    pass

            # if callbacks force us to leave they return None
            if model_info is None:
                # callback forced quit
                return saved_path

            # update the trains_out_model after the pre callbacks
            trains_out_model = model_info.model

            # check if object already has InputModel
            if trains_out_model is None:
                # noinspection PyProtectedMember
                in_model_id, model_uri = Model._local_model_to_id_uri.get(
                    model_info.local_model_id or model_info.local_model_path, (None, None))

                if not in_model_id:
                    # if we are overwriting a local file, try to load registered model
                    # if there is an output_uri, then by definition we will not overwrite previously stored models.
                    if not task.output_uri:
                        # noinspection PyBroadException
                        try:
                            in_model_id = InputModel.load_model(weights_url=model_info.local_model_path)
                            if in_model_id:
                                in_model_id = in_model_id.id

                                get_logger(TrainsFrameworkAdapter).info(
                                    "Found existing registered model id={} [{}] reusing it.".format(
                                        in_model_id, model_info.local_model_path))
                        except Exception:
                            in_model_id = None
                    else:
                        in_model_id = None

                trains_out_model = OutputModel(
                    task=task,
                    config_dict=config_obj if isinstance(config_obj, dict) else None,
                    config_text=config_obj if isinstance(config_obj, str) else None,
                    name=None if in_model_id else '{} - {}'.format(
                        task.name, model_name or Path(model_info.local_model_path).stem),
                    label_enumeration=task.get_labels_enumeration(),
                    framework=framework,
                    base_model_id=in_model_id
                )
                # # disable model reuse, let Model module try to find it for use
                # if model is not None:
                #     # noinspection PyBroadException
                #     try:
                #         ref_model = weakref.ref(model)
                #     except Exception:
                #         ref_model = None
                #     WeightsFileHandler._model_out_store_lookup[id(model)] = (trains_out_model, ref_model)

            model_info.model = trains_out_model
            # call post model callback functions
            for cb in WeightsFileHandler._model_post_callbacks.values():
                # noinspection PyBroadException
                try:
                    model_info = cb(WeightsFileHandler.CallbackType.save, model_info)
                except Exception:
                    pass
            trains_out_model = model_info.model
            target_filename = model_info.upload_filename

            # upload files if we found them, or just register the original path
            if trains_out_model.upload_storage_uri:
                if len(files) > 1:
                    trains_out_model.update_weights_package(
                        weights_filenames=files, auto_delete_file=False, target_filename=target_filename)
                else:
                    # create a copy of the stored file,
                    # protect against someone deletes/renames the file before async upload finish is done

                    # HACK: if pytorch-lightning is used, remove the temp '.part' file extension
                    if sys.modules.get('pytorch_lightning') and target_filename.lower().endswith('.part'):
                        target_filename = target_filename[:-len('.part')]
                    fd, temp_file = mkstemp(prefix='.clearml.upload_model_', suffix='.tmp')
                    os.close(fd)
                    shutil.copy(files[0], temp_file)
                    trains_out_model.update_weights(
                        weights_filename=temp_file, auto_delete_file=True, target_filename=target_filename,
                        update_comment=False)
            else:
                trains_out_model.update_weights(
                    weights_filename=None, register_uri=model_info.local_model_path, is_package=bool(len(files) > 1))

            # update back the internal Model lookup, and replace the local file with our file
            # noinspection PyProtectedMember
            Model._local_model_to_id_uri[model_info.local_model_id] = (
                trains_out_model.id, trains_out_model.url)

        except Exception as ex:
            get_logger(TrainsFrameworkAdapter).debug(str(ex))
        finally:
            WeightsFileHandler._model_store_lookup_lock.release()

        return saved_path


# clearml/binding/frameworks/base_bind.py

from abc import ABCMeta, abstractmethod

import six


@six.add_metaclass(ABCMeta)
class PatchBaseModelIO(object):
    """
    Base class for patched models

    :param __main_task: Task to run (Experiment)
    :type __main_task: Task
    :param __patched: True if the model is patched
    :type __patched: bool
    """
    @property
    @abstractmethod
    def __main_task(self):
        pass

    @property
    @abstractmethod
    def __patched(self):
        pass

    @staticmethod
    @abstractmethod
    def update_current_task(task, **kwargs):
        """
        Update the model task to run
        :param task: the experiment to do
        :type task: Task
        """
        pass

    @staticmethod
    @abstractmethod
    def _patch_model_io():
        """
        Patching the load and save functions
        """
        pass

    @staticmethod
    @abstractmethod
    def _save(original_fn, obj, f, *args, **kwargs):
        pass

    @staticmethod
    @abstractmethod
    def _load(original_fn, f, *args, **kwargs):
        pass


# clearml/binding/frameworks/catboost_bind.py

import sys

from pathlib2 import Path

import six

from ..frameworks import WeightsFileHandler, _Empty, _patched_call
from ..frameworks.base_bind import PatchBaseModelIO
from ..import_bind import PostImportHookPatching
from ...model import Framework


class PatchCatBoostModelIO(PatchBaseModelIO):
    __main_task = None
    __patched = None
    __callback_cls = None

    @staticmethod
    def update_current_task(task, **kwargs):
        PatchCatBoostModelIO.__main_task = task
        PatchCatBoostModelIO._patch_model_io()
        PostImportHookPatching.add_on_import("catboost", PatchCatBoostModelIO._patch_model_io)

    @staticmethod
    def _patch_model_io():
        if PatchCatBoostModelIO.__patched:
            return
        if "catboost" not in sys.modules:
            return
        PatchCatBoostModelIO.__patched = True
        # noinspection PyBroadException
        try:
            from catboost import CatBoost, CatBoostClassifier, CatBoostRegressor, CatBoostRanker

            CatBoost.save_model = _patched_call(CatBoost.save_model, PatchCatBoostModelIO._save)
            CatBoost.load_model = _patched_call(CatBoost.load_model, PatchCatBoostModelIO._load)
            PatchCatBoostModelIO.__callback_cls = PatchCatBoostModelIO._generate_training_callback_class()
            CatBoost.fit = _patched_call(CatBoost.fit, PatchCatBoostModelIO._fit)
            CatBoostClassifier.fit = _patched_call(CatBoostClassifier.fit, PatchCatBoostModelIO._fit)
            CatBoostRegressor.fit = _patched_call(CatBoostRegressor.fit, PatchCatBoostModelIO._fit)
            CatBoostRanker.fit = _patched_call(CatBoostRegressor.fit, PatchCatBoostModelIO._fit)
        except ImportError:
            pass
        except Exception:
            pass

    @staticmethod
    def _save(original_fn, obj, f, *args, **kwargs):
        # see https://catboost.ai/en/docs/concepts/python-reference_catboost_save_model
        ret = original_fn(obj, f, *args, **kwargs)
        if not PatchCatBoostModelIO.__main_task:
            return ret
        if isinstance(f, six.string_types):
            filename = f
        else:
            filename = None
        # give the model a descriptive name based on the file name
        # noinspection PyBroadException
        try:
            model_name = Path(filename).stem
        except Exception:
            model_name = None
        WeightsFileHandler.create_output_model(
            obj, filename, Framework.catboost, PatchCatBoostModelIO.__main_task, singlefile=True, model_name=model_name
        )
        return ret

    @staticmethod
    def _load(original_fn, f, *args, **kwargs):
        # see https://catboost.ai/en/docs/concepts/python-reference_catboost_load_model
        if isinstance(f, six.string_types):
            filename = f
        elif len(args) >= 1 and isinstance(args[0], six.string_types):
            filename = args[0]
        else:
            filename = None

        if not PatchCatBoostModelIO.__main_task:
            return original_fn(f, *args, **kwargs)

        # register input model
        empty = _Empty()
        model = original_fn(f, *args, **kwargs)
        WeightsFileHandler.restore_weights_file(empty, filename, Framework.catboost, PatchCatBoostModelIO.__main_task)
        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass
        return model

    @staticmethod
    def _fit(original_fn, obj, *args, **kwargs):
        callbacks = kwargs.get("callbacks") or []
        kwargs["callbacks"] = callbacks + [PatchCatBoostModelIO.__callback_cls(task=PatchCatBoostModelIO.__main_task)]
        return original_fn(obj, *args, **kwargs)

    @staticmethod
    def _generate_training_callback_class():
        class ClearMLCallback:
            def __init__(self, task):
                self._logger = task.get_logger()

            def after_iteration(self, info):
                info = vars(info)
                iteration = info.get("iteration")
                for title, metric in (info.get("metrics") or {}).items():
                    for series, log in metric.items():
                        value = log[-1]
                        self._logger.report_scalar(title=title, series=series, value=value, iteration=iteration)
                return True

        return ClearMLCallback


# clearml/binding/frameworks/fastai_bind.py

import sys

import numpy as np

from . import _patched_call
from .tensorflow_bind import WeightsGradientHistHelper
from ..import_bind import PostImportHookPatching
from ...debugging.log import LoggerRoot


class PatchFastai(object):
    __metrics_names = None  # TODO: STORE ON OBJECT OR IN LOOKUP BASED ON OBJECT ID
    __main_task = None

    @staticmethod
    def update_current_task(task, **_):
        PatchFastai.__main_task = task
        PatchFastai._patch_model_callback()
        PostImportHookPatching.add_on_import("fastai", PatchFastai._patch_model_callback)

    @staticmethod
    def _patch_model_callback():
        # if you have tensroboard, we assume you use TesnorboardLogger, which we catch, so no need to patch.
        if "tensorboard" in sys.modules:
            return

        if "fastai" in sys.modules:
            try:
                from fastai.basic_train import Recorder

                Recorder.on_batch_end = _patched_call(Recorder.on_batch_end, PatchFastai._on_batch_end)
                Recorder.on_backward_end = _patched_call(Recorder.on_backward_end, PatchFastai._on_backward_end)
                Recorder.on_epoch_end = _patched_call(Recorder.on_epoch_end, PatchFastai._on_epoch_end)
                Recorder.on_train_begin = _patched_call(Recorder.on_train_begin, PatchFastai._on_train_begin)
            except ImportError:
                pass
            except Exception as ex:
                LoggerRoot.get_base_logger(PatchFastai).debug(str(ex))

    @staticmethod
    def _on_train_begin(original_fn, recorder, *args, **kwargs):
        original_fn(recorder, *args, **kwargs)
        if not PatchFastai.__main_task:
            return
        # noinspection PyBroadException
        try:
            PatchFastai.__metrics_names = ["train_loss"] if recorder.no_val else ["train_loss", "valid_loss"]
            PatchFastai.__metrics_names += recorder.metrics_names
        except Exception:
            pass

    @staticmethod
    def _on_backward_end(original_fn, recorder, *args, **kwargs):
        def count_zeros(gradient):
            n = gradient.data.data.cpu().numpy()
            return n.size - n.count_nonzero()

        original_fn(recorder, *args, **kwargs)

        if not PatchFastai.__main_task:
            return

        # noinspection PyBroadException
        try:
            gradients = [
                x.grad.clone().detach().cpu() for x in recorder.learn.model.parameters() if x.grad is not None
            ]
            if len(gradients) == 0:
                return

            # TODO: Check computation!
            gradient_stats = np.array([
                (x.data.norm(), count_zeros(x), x.data.mean(), x.data.median(), x.data.max(), x.data.min())
                for x in gradients])
            stats_report = dict(
                avg_norm=np.mean(gradient_stats[:, 0]),
                median_norm=np.median(gradient_stats[:, 0]),
                max_norm=np.max(gradient_stats[:, 0]),
                min_norm=np.min(gradient_stats[:, 0]),
                num_zeros=gradient_stats[:, 1].sum(),
                avg_gradient=gradient_stats[:, 2].mean(),
                median_gradient=gradient_stats[:, 3].median(),
                max_gradient=gradient_stats[:, 4].max(),
                min_gradient=gradient_stats[:, 5].min(),
            )

            logger = PatchFastai.__main_task.get_logger()
            iteration = kwargs.get("iteration", 0)
            for name, val in stats_report.items():
                logger.report_scalar(title="model_stats_gradients", series=name, value=val, iteration=iteration)
        except Exception:
            pass

    @staticmethod
    def _on_epoch_end(original_fn, recorder, *args, **kwargs):
        original_fn(recorder, *args, **kwargs)
        if not PatchFastai.__main_task:
            return

        # noinspection PyBroadException
        try:
            logger = PatchFastai.__main_task.get_logger()
            iteration = kwargs.get("iteration")
            for series, value in zip(
                    PatchFastai.__metrics_names,
                    [kwargs.get("smooth_loss")] + kwargs.get("last_metrics", []),
            ):
                logger.report_scalar(title="metrics", series=series, value=value, iteration=iteration)
            PatchFastai.__main_task.flush()
        except Exception:
            pass

    @staticmethod
    def _on_batch_end(original_fn, recorder, *args, **kwargs):
        original_fn(recorder, *args, **kwargs)
        if not PatchFastai.__main_task:
            return

        # noinspection PyBroadException
        try:
            if kwargs.get("iteration") == 0 or not kwargs.get("train"):
                return

            logger = PatchFastai.__main_task.get_logger()
            logger.report_scalar(
                title="metrics",
                series="train_loss",
                value=kwargs.get("last_loss", 0),
                iteration=kwargs.get("iteration", 0)
            )
            gradient_hist_helper = WeightsGradientHistHelper(logger)
            iteration = kwargs.get("iteration")
            params = [
                (name, values.clone().detach().cpu())
                for (name, values) in recorder.model.named_parameters()
            ]
            for (name, values) in params:
                gradient_hist_helper.add_histogram(
                    title="model_weights",
                    series="model_weights/" + name,
                    step=iteration,
                    hist_data=values,
                )
        except Exception:
            pass


# clearml/binding/frameworks/lightgbm_bind.py

import sys

import six
from pathlib2 import Path

from ..frameworks.base_bind import PatchBaseModelIO
from ..frameworks import _patched_call, WeightsFileHandler, _Empty
from ..import_bind import PostImportHookPatching
from ...config import running_remotely
from ...model import Framework


class PatchLIGHTgbmModelIO(PatchBaseModelIO):
    __main_task = None
    __patched = None

    @staticmethod
    def update_current_task(task, **kwargs):
        PatchLIGHTgbmModelIO.__main_task = task
        PatchLIGHTgbmModelIO._patch_model_io()
        PostImportHookPatching.add_on_import('lightgbm', PatchLIGHTgbmModelIO._patch_model_io)

    @staticmethod
    def _patch_model_io():
        if PatchLIGHTgbmModelIO.__patched:
            return

        if 'lightgbm' not in sys.modules:
            return
        PatchLIGHTgbmModelIO.__patched = True
        # noinspection PyBroadException
        try:
            import lightgbm as lgb  # noqa

            lgb.Booster.save_model = _patched_call(lgb.Booster.save_model, PatchLIGHTgbmModelIO._save)
            lgb.train = _patched_call(lgb.train, PatchLIGHTgbmModelIO._train)
            lgb.Booster = _patched_call(lgb.Booster, PatchLIGHTgbmModelIO._load)
        except ImportError:
            pass
        except Exception:
            pass

    @staticmethod
    def _save(original_fn, obj, f, *args, **kwargs):
        ret = original_fn(obj, f, *args, **kwargs)
        if not PatchLIGHTgbmModelIO.__main_task:
            return ret

        if isinstance(f, six.string_types):
            filename = f
        elif hasattr(f, 'name'):
            filename = f.name
            # noinspection PyBroadException
            try:
                f.flush()
            except Exception:
                pass
        else:
            filename = None

        # give the model a descriptive name based on the file name
        # noinspection PyBroadException
        try:
            model_name = Path(filename).stem
        except Exception:
            model_name = None
        WeightsFileHandler.create_output_model(obj, filename, Framework.lightgbm, PatchLIGHTgbmModelIO.__main_task,
                                               singlefile=True, model_name=model_name)
        return ret

    @staticmethod
    def _load(original_fn, model_file, *args, **kwargs):
        if isinstance(model_file, six.string_types):
            filename = model_file
        elif hasattr(model_file, 'name'):
            filename = model_file.name
        elif len(args) == 1 and isinstance(args[0], six.string_types):
            filename = args[0]
        else:
            filename = None

        if not PatchLIGHTgbmModelIO.__main_task:
            return original_fn(model_file, *args, **kwargs)

        # register input model
        empty = _Empty()
        # Hack: disabled
        if False and running_remotely():
            filename = WeightsFileHandler.restore_weights_file(empty, filename, Framework.xgboost,
                                                               PatchLIGHTgbmModelIO.__main_task)
            model = original_fn(model_file=filename or model_file, *args, **kwargs)
        else:
            # try to load model before registering, in case we fail
            model = original_fn(model_file=model_file, *args, **kwargs)
            WeightsFileHandler.restore_weights_file(empty, filename, Framework.lightgbm,
                                                    PatchLIGHTgbmModelIO.__main_task)

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass
        return model

    @staticmethod
    def _train(original_fn, *args, **kwargs):
        def trains_lightgbm_callback():
            def callback(env):
                # logging the results to scalars section
                # noinspection PyBroadException
                try:
                    logger = PatchLIGHTgbmModelIO.__main_task.get_logger()
                    iteration = env.iteration
                    for data_title, data_series, value, _ in env.evaluation_result_list:
                        logger.report_scalar(title=data_title, series=data_series, value="{:.6f}".format(value),
                                             iteration=iteration)
                except Exception:
                    pass
            return callback

        kwargs.setdefault("callbacks", []).append(trains_lightgbm_callback())
        ret = original_fn(*args, **kwargs)
        if not PatchLIGHTgbmModelIO.__main_task:
            return ret
        params = args[0] if args else kwargs.get('params', {})
        for k, v in params.items():
            if isinstance(v, set):
                params[k] = list(v)
        if params:
            PatchLIGHTgbmModelIO.__main_task.connect(params)
        return ret


# clearml/binding/frameworks/megengine_bind.py

import sys

import six
from pathlib2 import Path

from ..frameworks.base_bind import PatchBaseModelIO
from ..frameworks import _patched_call, WeightsFileHandler, _Empty
from ..import_bind import PostImportHookPatching
from ...model import Framework


class PatchMegEngineModelIO(PatchBaseModelIO):
    __main_task = None
    __patched = None

    # __patched_lightning = None

    @staticmethod
    def update_current_task(task, **_):
        PatchMegEngineModelIO.__main_task = task
        PatchMegEngineModelIO._patch_model_io()
        PostImportHookPatching.add_on_import(
            'megengine', PatchMegEngineModelIO._patch_model_io
        )

    @staticmethod
    def _patch_model_io():
        if PatchMegEngineModelIO.__patched:
            return

        if 'megengine' not in sys.modules:
            return

        PatchMegEngineModelIO.__patched = True

        # noinspection PyBroadException
        try:
            import megengine as mge  # noqa
            mge.save = _patched_call(mge.save, PatchMegEngineModelIO._save)
            mge.load = _patched_call(mge.load, PatchMegEngineModelIO._load)

            # no need to worry about recursive calls, _patched_call takes care of that  # noqa
            if hasattr(mge, 'serialization') and hasattr(mge.serialization, 'save'):  # noqa
                mge.serialization.save = _patched_call(
                    mge.serialization.save, PatchMegEngineModelIO._save
                )
            if hasattr(mge, 'serialization') and hasattr(mge.serialization, 'load'):  # noqa
                mge.serialization.load = _patched_call(
                    mge.serialization.load, PatchMegEngineModelIO._load,
                )
        except ImportError:
            pass
        except Exception:
            pass

    @staticmethod
    def _save(original_fn, obj, f, *args, **kwargs):
        ret = original_fn(obj, f, *args, **kwargs)

        # if there is no main task or this is a nested call
        if not PatchMegEngineModelIO.__main_task:
            return ret

        # noinspection PyBroadException
        try:
            if isinstance(f, six.string_types):
                filename = f
            elif hasattr(f, 'as_posix'):
                filename = f.as_posix()
            elif hasattr(f, 'name'):
                # noinspection PyBroadException
                try:
                    f.flush()
                except Exception:
                    pass

                if not isinstance(f.name, six.string_types):
                    # Probably a BufferedRandom object that has no meaningful name (still no harm flushing)  # noqa
                    return ret

                filename = f.name
            else:
                filename = None
        except Exception:
            filename = None

        # give the model a descriptive name based on the file name
        # noinspection PyBroadException
        try:
            model_name = Path(filename).stem if filename is not None else None
        except Exception:
            model_name = None

        WeightsFileHandler.create_output_model(
            obj, filename, Framework.megengine,
            PatchMegEngineModelIO.__main_task,
            singlefile=True, model_name=model_name,
        )

        return ret

    @staticmethod
    def _load(original_fn, f, *args, **kwargs):
        # if there is no main task or this is a nested call
        if not PatchMegEngineModelIO.__main_task:
            return original_fn(f, *args, **kwargs)

        # noinspection PyBroadException
        try:
            if isinstance(f, six.string_types):
                filename = f
            elif hasattr(f, 'as_posix'):
                filename = f.as_posix()
            elif hasattr(f, 'name'):
                filename = f.name
            else:
                filename = None
        except Exception:
            filename = None

        # register input model
        empty = _Empty()
        # try to load model before registering, in case we fail
        model = original_fn(f, *args, **kwargs)
        WeightsFileHandler.restore_weights_file(
            empty, filename, Framework.megengine,
            PatchMegEngineModelIO.__main_task
        )

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass

        return model

    @staticmethod
    def _load_from_obj(original_fn, obj, f, *args, **kwargs):
        # if there is no main task or this is a nested call
        if not PatchMegEngineModelIO.__main_task:
            return original_fn(obj, f, *args, **kwargs)

        # noinspection PyBroadException
        try:
            if isinstance(f, six.string_types):
                filename = f
            elif hasattr(f, 'as_posix'):
                filename = f.as_posix()
            elif hasattr(f, 'name'):
                filename = f.name
            else:
                filename = None
        except Exception:
            filename = None

        # register input model
        empty = _Empty()
        # try to load model before registering, in case we fail
        model = original_fn(obj, f, *args, **kwargs)
        WeightsFileHandler.restore_weights_file(
            empty, filename, Framework.megengine,
            PatchMegEngineModelIO.__main_task,
        )

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass

        return model


# clearml/binding/frameworks/pytorch_bind.py

import sys

import six
from pathlib2 import Path

from ...binding.frameworks.base_bind import PatchBaseModelIO
from ..frameworks import _patched_call, WeightsFileHandler, _Empty
from ..import_bind import PostImportHookPatching
from ...config import running_remotely
from ...model import Framework


class PatchPyTorchModelIO(PatchBaseModelIO):
    __main_task = None
    __patched = None
    __patched_lightning = None

    @staticmethod
    def update_current_task(task, **_):
        PatchPyTorchModelIO.__main_task = task
        PatchPyTorchModelIO._patch_model_io()
        PatchPyTorchModelIO._patch_lightning_io()
        PostImportHookPatching.add_on_import('torch', PatchPyTorchModelIO._patch_model_io)
        PostImportHookPatching.add_on_import('pytorch_lightning', PatchPyTorchModelIO._patch_lightning_io)

    @staticmethod
    def _patch_model_io():
        if PatchPyTorchModelIO.__patched:
            return

        if 'torch' not in sys.modules:
            return

        PatchPyTorchModelIO.__patched = True

        # noinspection PyBroadException
        try:
            import torch  # noqa
            torch.save = _patched_call(torch.save, PatchPyTorchModelIO._save)
            torch.load = _patched_call(torch.load, PatchPyTorchModelIO._load)

            # no need to worry about recursive calls, _patched_call takes care of that
            if hasattr(torch, 'serialization') and hasattr(torch.serialization, '_save'):
                torch.serialization._save = _patched_call(
                    torch.serialization._save, PatchPyTorchModelIO._save)  # noqa
            if hasattr(torch, 'serialization') and hasattr(torch.serialization, '_load'):
                torch.serialization._load = _patched_call(
                    torch.serialization._load, PatchPyTorchModelIO._load)  # noqa
            if hasattr(torch, 'serialization') and hasattr(torch.serialization, '_legacy_save'):
                torch.serialization._legacy_save = _patched_call(
                    torch.serialization._legacy_save, PatchPyTorchModelIO._save)  # noqa
            if hasattr(torch, 'serialization') and hasattr(torch.serialization, '_legacy_load'):
                torch.serialization._legacy_load = _patched_call(
                    torch.serialization._legacy_load, PatchPyTorchModelIO._load)  # noqa
        except ImportError:
            pass
        except Exception:
            pass  # print('Failed patching pytorch')

    @staticmethod
    def _patch_lightning_io():
        if PatchPyTorchModelIO.__patched_lightning:
            return

        if 'pytorch_lightning' not in sys.modules:
            return

        PatchPyTorchModelIO.__patched_lightning = True

        # noinspection PyBroadException
        try:
            import pytorch_lightning  # noqa

            pytorch_lightning.trainer.Trainer.save_checkpoint = _patched_call(
                pytorch_lightning.trainer.Trainer.save_checkpoint, PatchPyTorchModelIO._save)  # noqa

            pytorch_lightning.trainer.Trainer.restore = _patched_call(
                pytorch_lightning.trainer.Trainer.restore, PatchPyTorchModelIO._load_from_obj)  # noqa
        except ImportError:
            pass
        except Exception:
            pass

        # noinspection PyBroadException
        try:
            import pytorch_lightning  # noqa

            # noinspection PyUnresolvedReferences
            pytorch_lightning.trainer.connectors.checkpoint_connector.CheckpointConnector.save_checkpoint = \
                _patched_call(
                    pytorch_lightning.trainer.connectors.checkpoint_connector.CheckpointConnector.save_checkpoint,
                    PatchPyTorchModelIO._save)  # noqa

            # noinspection PyUnresolvedReferences
            pytorch_lightning.trainer.connectors.checkpoint_connector.CheckpointConnector.restore = \
                _patched_call(
                    pytorch_lightning.trainer.connectors.checkpoint_connector.CheckpointConnector.restore,
                    PatchPyTorchModelIO._load_from_obj)  # noqa
        except ImportError:
            pass
        except Exception:
            pass

    @staticmethod
    def _save(original_fn, obj, f, *args, **kwargs):
        ret = original_fn(obj, f, *args, **kwargs)

        # if there is no main task or this is a nested call
        if not PatchPyTorchModelIO.__main_task:
            return ret

        # pytorch-lightning check if rank is zero
        if hasattr(obj, 'is_global_zero'):
            if not obj.is_global_zero:
                return ret
        elif hasattr(obj, 'trainer') and hasattr(obj.trainer, 'is_global_zero'):
            if not obj.trainer.is_global_zero:
                return ret

        # noinspection PyBroadException
        try:
            if isinstance(f, six.string_types):
                filename = f
            elif hasattr(f, 'as_posix'):
                filename = f.as_posix()
            elif hasattr(f, 'name'):
                # noinspection PyBroadException
                try:
                    f.flush()
                except Exception:
                    pass

                if not isinstance(f.name, six.string_types):
                    # Probably a BufferedRandom object that has no meaningful name (still no harm flushing)
                    return ret

                filename = f.name
            else:
                filename = None
        except Exception:
            filename = None

        # give the model a descriptive name based on the file name
        # noinspection PyBroadException
        try:
            model_name = Path(filename).stem if filename is not None else None
        except Exception:
            model_name = None

        WeightsFileHandler.create_output_model(
            obj, filename, Framework.pytorch, PatchPyTorchModelIO.__main_task, singlefile=True, model_name=model_name)

        return ret

    @staticmethod
    def _load(original_fn, f, *args, **kwargs):
        # if there is no main task or this is a nested call
        if not PatchPyTorchModelIO.__main_task:
            return original_fn(f, *args, **kwargs)

        # noinspection PyBroadException
        try:
            if isinstance(f, six.string_types):
                filename = f
            elif hasattr(f, 'as_posix'):
                filename = f.as_posix()
            elif hasattr(f, 'name'):
                filename = f.name
            else:
                filename = None
        except Exception:
            filename = None

        # register input model
        empty = _Empty()
        # Hack: disabled
        if False and running_remotely():
            filename = WeightsFileHandler.restore_weights_file(
                empty, filename, Framework.pytorch, PatchPyTorchModelIO.__main_task)
            model = original_fn(filename or f, *args, **kwargs)
        else:
            # try to load model before registering, in case we fail
            model = original_fn(f, *args, **kwargs)
            WeightsFileHandler.restore_weights_file(
                empty, filename, Framework.pytorch, PatchPyTorchModelIO.__main_task)

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass

        return model

    @staticmethod
    def _load_from_obj(original_fn, obj, f, *args, **kwargs):
        # if there is no main task or this is a nested call
        if not PatchPyTorchModelIO.__main_task:
            return original_fn(obj, f, *args, **kwargs)

        # noinspection PyBroadException
        try:
            if isinstance(f, six.string_types):
                filename = f
            elif hasattr(f, 'as_posix'):
                filename = f.as_posix()
            elif hasattr(f, 'name'):
                filename = f.name
            else:
                filename = None
        except Exception:
            filename = None

        # register input model
        empty = _Empty()
        # Hack: disabled
        if False and running_remotely():
            filename = WeightsFileHandler.restore_weights_file(
                empty, filename, Framework.pytorch, PatchPyTorchModelIO.__main_task)
            model = original_fn(obj, filename or f, *args, **kwargs)
        else:
            # try to load model before registering, in case we fail
            model = original_fn(obj, f, *args, **kwargs)
            WeightsFileHandler.restore_weights_file(
                empty, filename, Framework.pytorch, PatchPyTorchModelIO.__main_task)

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass

        return model


# clearml/binding/frameworks/tensorflow_bind.py

import base64
import os
import sys
import threading
from collections import defaultdict
from functools import partial
from io import BytesIO
from mimetypes import guess_extension
from tempfile import mkstemp
from typing import Any, Union

import numpy as np
import six
from PIL import Image

from ...debugging.log import LoggerRoot
from ..frameworks import _patched_call, WeightsFileHandler, _Empty
from ..import_bind import PostImportHookPatching
from ...config import running_remotely
from ...model import InputModel, OutputModel, Framework

try:
    from google.protobuf.json_format import MessageToDict  # noqa
except ImportError:
    MessageToDict = None

try:
    from PIL import GifImagePlugin  # noqa
except ImportError:
    pass


class TensorflowBinding(object):
    @classmethod
    def update_current_task(cls, task, patch_reporting=True, patch_model_io=True):
        if not task:
            IsTensorboardInit.clear_tensorboard_used()

        EventTrainsWriter.update_current_task(task)

        if patch_reporting:
            PatchSummaryToEventTransformer.update_current_task(task)
            PatchTensorFlowEager.update_current_task(task)

        if patch_model_io:
            PatchKerasModelIO.update_current_task(task)
            PatchTensorflowModelIO.update_current_task(task)
            PatchTensorflow2ModelIO.update_current_task(task)


class IsTensorboardInit(object):
    _tensorboard_initialized = False

    @classmethod
    def tensorboard_used(cls):
        return cls._tensorboard_initialized

    @classmethod
    def set_tensorboard_used(cls):
        cls._tensorboard_initialized = True

    @classmethod
    def clear_tensorboard_used(cls):
        cls._tensorboard_initialized = False

    @staticmethod
    def _patched_tb__init__(original_init, self, *args, **kwargs):
        IsTensorboardInit._tensorboard_initialized = True
        return original_init(self, *args, **kwargs)


# noinspection PyProtectedMember
class WeightsGradientHistHelper(object):
    def __init__(self, logger, report_freq=100, histogram_update_freq_multiplier=10, histogram_granularity=50):
        self._logger = logger
        self.report_freq = report_freq
        self._histogram_granularity = histogram_granularity
        self._histogram_update_freq_multiplier = histogram_update_freq_multiplier
        self._histogram_update_call_counter = 0
        self._hist_report_cache = {}
        self._hist_x_granularity = 50

    @staticmethod
    def _sample_histograms(_hist_iters, _histogram_granularity):
        # re-sample history based on distribution of samples across time (steps)
        ratio = ((_hist_iters[-1] - _hist_iters[_histogram_granularity]) /
                 (_hist_iters[_histogram_granularity - 1] - _hist_iters[0])) if \
            _hist_iters.size > _histogram_granularity else 0.
        cur_idx_below = np.arange(0, min(_hist_iters.size, _histogram_granularity - 1))
        np.random.shuffle(cur_idx_below)
        cur_idx_below = cur_idx_below[:int(_histogram_granularity * (1.0 - ratio / (1 + ratio)) + 0.5)]
        if ratio > 0.0:
            cur_idx_above = np.arange(_histogram_granularity - 1, _hist_iters.size)
            np.random.shuffle(cur_idx_above)
            cur_idx_above = cur_idx_above[:int(_histogram_granularity * ratio / (1 + ratio))]
        else:
            cur_idx_above = np.array([])
        _cur_idx = np.unique(np.sort(np.concatenate((cur_idx_below, cur_idx_above)).astype(np.int)))
        return _cur_idx

    def add_histogram(self, title, series, step, hist_data):
        # only collect histogram every specific interval
        self._histogram_update_call_counter += 1
        if self._histogram_update_call_counter % self.report_freq != 0 or \
                self._histogram_update_call_counter < self.report_freq - 1:
            return None

        if isinstance(hist_data, dict):
            pass
        elif isinstance(hist_data, np.ndarray) and len(hist_data.shape) == 2 and np.atleast_2d(hist_data).shape[1] == 3:
            # prepare the dictionary, assume numpy
            # hist_data['bucketLimit'] is the histogram bucket right side limit, meaning X axis
            # hist_data['bucket'] is the histogram height, meaning the Y axis
            # notice hist_data[:, 1] is the right side limit, for backwards compatibility we take the left side
            hist_data = {'bucketLimit': hist_data[:, 0].tolist(), 'bucket': hist_data[:, 2].tolist()}
        else:
            # assume we have to do the histogram on the data
            hist_data = np.histogram(hist_data, bins=32)
            hist_data = {'bucketLimit': hist_data[1].tolist(), 'bucket': hist_data[0].tolist()}

        self._add_histogram(title=title, series=series, step=step, hist_data=hist_data)

    def _add_histogram(self, title, series, step, hist_data):
        # only collect histogram every specific interval
        self._histogram_update_call_counter += 1
        if self._histogram_update_call_counter % self.report_freq != 0 or \
                self._histogram_update_call_counter < self.report_freq - 1:
            return None

        # generate forward matrix of the histograms
        # Y-axis (rows) is iteration (from 0 to current Step)
        # X-axis averaged bins (conformed sample 'bucketLimit')
        # Z-axis actual value (interpolated 'bucket')
        step = EventTrainsWriter._fix_step_counter(title, series, step)

        # get histograms from cache
        hist_list, hist_iters, minmax = self._hist_report_cache.get((title, series), ([], np.array([]), None))

        # resample data so we are always constrained in number of histogram we keep
        if hist_iters.size >= self._histogram_granularity ** 2:
            idx = self._sample_histograms(hist_iters, self._histogram_granularity)
            hist_iters = hist_iters[idx]
            hist_list = [hist_list[i] for i in idx]

        # check if current sample is not already here (actually happens some times)
        if step in hist_iters:
            return None

        # add current sample, if not already here
        hist_iters = np.append(hist_iters, step)
        # hist_data['bucketLimit'] is the histogram bucket right side limit, meaning X axis
        # hist_data['bucket'] is the histogram height, meaning the Y axis
        hist = np.array(list(zip(hist_data['bucketLimit'], hist_data['bucket'])), dtype=np.float32)
        hist = hist[~np.isinf(hist[:, 0]), :]
        hist_list.append(hist)
        # keep track of min/max values of histograms (for later re-binning)
        if minmax is None:
            minmax = hist[:, 0].min(), hist[:, 0].max()
        else:
            # noinspection PyUnresolvedReferences
            minmax = min(minmax[0], hist[:, 0].min()), max(minmax[1], hist[:, 0].max())

        # update the cache
        self._hist_report_cache[(title, series)] = hist_list, hist_iters, minmax

        # only report histogram every specific interval, but do report the first few, so you know there are histograms
        if hist_iters.size < 1 or (hist_iters.size >= self._histogram_update_freq_multiplier and
                                   hist_iters.size % self._histogram_update_freq_multiplier != 0):
            return None

        # resample histograms on a unified bin axis +- epsilon
        _epsilon = abs((minmax[1] - minmax[0])/float(self._hist_x_granularity))
        if _epsilon == 0:
            _epsilon = 0.01
        _minmax = minmax[0] - _epsilon, minmax[1] + _epsilon
        prev_xedge = np.arange(start=_minmax[0],
                               step=(_minmax[1] - _minmax[0]) / float(self._hist_x_granularity - 2), stop=_minmax[1])
        # uniformly select histograms and the last one
        cur_idx = self._sample_histograms(hist_iters, self._histogram_granularity)
        report_hist = np.zeros(shape=(len(cur_idx), prev_xedge.size), dtype=np.float32)
        for i, n in enumerate(cur_idx):
            h = hist_list[n]
            report_hist[i, :] = np.interp(prev_xedge, h[:, 0], h[:, 1], right=0, left=0)
        yedges = hist_iters[cur_idx]
        xedges = prev_xedge

        # if only a single line make, add another zero line, for the scatter plot to draw
        if report_hist.shape[0] < 2:
            report_hist = np.vstack((np.zeros_like(report_hist), report_hist))

        # create 3d line (scatter) of histograms
        skipx = max(1, int(xedges.size / 10))
        skipy = max(1, int(yedges.size / 10))
        xlabels = ['%.2f' % v if i % skipx == 0 else '' for i, v in enumerate(xedges[:-1])]
        ylabels = [str(int(v)) if i % skipy == 0 else '' for i, v in enumerate(yedges)]
        self._logger.report_surface(
            title=title,
            series=series,
            iteration=0,
            xaxis=' ',
            yaxis='iteration',
            xlabels=xlabels,
            ylabels=ylabels,
            matrix=report_hist,
            camera=(-0.1, +1.3, 1.4))


# noinspection PyMethodMayBeStatic,PyProtectedMember,SpellCheckingInspection
class EventTrainsWriter(object):
    """
    TF SummaryWriter implementation that converts the tensorboard's summary into
    ClearML events and reports the events (metrics) for an ClearML task (logger).
    """
    __main_task = None
    _add_lock = threading.RLock()
    _series_name_lookup = {}

    # store all the created tensorboard writers in the system
    # this allows us to as weather a certain tile/series already exist on some EventWriter
    # and if it does, then we add to the series name the last token from the logdir
    # (so we can differentiate between the two)
    # key, value: key=hash(title, graph), value=EventTrainsWriter._id
    _title_series_writers_lookup = {}
    _event_writers_id_to_logdir = {}

    # Protect against step (iteration) reuse, for example,
    # steps counter inside an epoch, but wrapping around when epoch ends
    # i.e. step = 0..100 then epoch ends and again step = 0..100
    # We store the first report per title/series combination, and if wraparound occurs
    # we synthetically continue to increase the step/iteration based on the previous epoch counter
    # example: _title_series_wraparound_counter[('title', 'series')] =
    #           {'first_step':None, 'last_step':None, 'adjust_counter':0,}
    _title_series_wraparound_counter = {}

    @property
    def variants(self):
        return self._variants

    def prepare_report(self):
        return self.variants.copy()

    def tag_splitter(self, tag, num_split_parts, split_char='/', join_char='_', default_title='variant',
                     logdir_header='series', auto_reduce_num_split=False, force_add_prefix=None):
        """
        Split a tf.summary tag line to variant and metric.
        Variant is the first part of the split tag, metric is the second.
        :param str tag:
        :param int num_split_parts:
        :param str split_char: a character to split the tag on
        :param str join_char: a character to join the the splits
        :param str default_title: variant to use in case no variant can be inferred automatically
        :param str logdir_header: if 'series_last' then series=header: series, if 'series then series=series :header,
            if 'title_last' then title=header title, if 'title' then title=title header
        :param bool auto_reduce_num_split: if True and the tag is split for less parts then requested,
            then requested number of split parts is adjusted.
        :param str force_add_prefix: always add the prefix to the series name
        :return: (str, str) variant and metric
        """
        splitted_tag = tag.split(split_char)
        if auto_reduce_num_split and num_split_parts > len(splitted_tag) - 1:
            num_split_parts = max(1, len(splitted_tag) - 1)
        series = join_char.join(splitted_tag[-num_split_parts:])
        title = join_char.join(splitted_tag[:-num_split_parts]) or default_title

        if force_add_prefix:
            series = str(force_add_prefix)+series

        # check if we already decided that we need to change the title/series
        graph_id = hash((title, series))
        if graph_id in self._graph_name_lookup:
            return self._graph_name_lookup[graph_id]

        # check if someone other than us used this combination
        with self._add_lock:
            event_writer_id = self._title_series_writers_lookup.get(graph_id, None)
            if not event_writer_id:
                # put us there
                self._title_series_writers_lookup[graph_id] = self._id
            elif event_writer_id != self._id:
                # if there is someone else, change our series name and store us
                org_series = series
                org_title = title
                other_logdir = self._event_writers_id_to_logdir[event_writer_id]
                split_logddir = self._logdir.split('/')
                unique_logdir = set(split_logddir) - set(other_logdir.split('/'))
                header = '/'.join(s for s in split_logddir if s in unique_logdir)
                if logdir_header == 'series_last':
                    series = header + ': ' + series
                elif logdir_header == 'series':
                    series = series + ' :' + header
                elif logdir_header == 'title':
                    title = title + ' ' + header
                else:  # logdir_header == 'title_last':
                    title = header + ' ' + title
                graph_id = hash((title, series))
                # check if for some reason the new series is already occupied
                new_event_writer_id = self._title_series_writers_lookup.get(graph_id)
                if new_event_writer_id is not None and new_event_writer_id != self._id:
                    # well that's about it, nothing else we could do
                    if logdir_header == 'series_last':
                        series = str(self._logdir) + ': ' + org_series
                    elif logdir_header == 'series':
                        series = org_series + ' :' + str(self._logdir)
                    elif logdir_header == 'title':
                        title = org_title + ' ' + str(self._logdir)
                    else:  # logdir_header == 'title_last':
                        title = str(self._logdir) + ' ' + org_title
                    graph_id = hash((title, series))

                self._title_series_writers_lookup[graph_id] = self._id

        # store for next time
        self._graph_name_lookup[graph_id] = (title, series)
        return title, series

    def __init__(self, logger, logdir=None, report_freq=100, image_report_freq=None,
                 histogram_update_freq_multiplier=10, histogram_granularity=50, max_keep_images=None):
        """
        Create a compatible ClearML backend to the TensorFlow SummaryToEventTransformer
        Everything will be serialized directly to the ClearML backend, instead of to the standard TF FileWriter

        :param logger: The task.logger to use for sending the metrics (def: task.get_logger())
        :param report_freq: How often to update the statistics values
        :param image_report_freq: How often to upload images (step % image_update_freq == 0)
        :param histogram_update_freq_multiplier: How often to upload histogram
               (step//update_freq) % histogram_update_freq_multiplier == 0
        :param histogram_granularity: How many histograms (lines) to display in the 3d histogram plot
        :param max_keep_images: Maximum number of images to save before starting to reuse files (per title/metric pair)
        """
        # We are the events_writer, so that's what we'll pass
        IsTensorboardInit.set_tensorboard_used()
        self._logdir = logdir or ('unknown %d' % len(self._event_writers_id_to_logdir))
        # conform directory structure to unix
        if os.path.sep == '\\':
            self._logdir = self._logdir.replace('\\', '/')
        self._id = hash(self._logdir)
        self._event_writers_id_to_logdir[self._id] = self._logdir
        self.max_keep_images = max_keep_images
        self.report_freq = report_freq
        self.image_report_freq = image_report_freq if image_report_freq else report_freq
        self.histogram_granularity = histogram_granularity
        self.histogram_update_freq_multiplier = histogram_update_freq_multiplier
        self._histogram_update_call_counter = 0
        self._logger = logger
        self._visualization_mode = 'RGB'  # 'BGR'
        self._variants = defaultdict(lambda: ())
        self._scalar_report_cache = {}
        self._hist_report_cache = {}
        self._hist_x_granularity = 50
        self._max_step = 0
        self._graph_name_lookup = {}
        self._generic_tensor_type_name_lookup = {}
        self._grad_helper = WeightsGradientHistHelper(
            logger=logger,
            report_freq=report_freq,
            histogram_update_freq_multiplier=histogram_update_freq_multiplier,
            histogram_granularity=histogram_granularity
        )

    def _decode_image(self, img_str, width=None, height=None, color_channels=None):
        # noinspection PyBroadException
        try:
            if isinstance(img_str, bytes):
                imdata = img_str
            else:
                imdata = base64.b64decode(img_str)
            output = BytesIO(imdata)
            im = Image.open(output)
            # if this is a GIF store as is
            if getattr(im, 'is_animated', None):
                output.close()
                fd, temp_file = mkstemp(
                    suffix=guess_extension(im.get_format_mimetype()) if hasattr(im, 'get_format_mimetype')
                    else ".{}".format(str(im.format).lower())
                )
                os.write(fd, imdata)
                os.close(fd)
                return temp_file

            image = np.asarray(im)
            output.close()
            if height is not None and height > 0 and width is not None and width > 0:
                # noinspection PyArgumentList
                val = image.reshape(height, width, -1).astype(np.uint8)
            else:
                val = image.astype(np.uint8)
            if val.ndim == 3 and val.shape[2] == 3:
                if self._visualization_mode == 'BGR':
                    val = val[:, :, [2, 1, 0]]
                else:
                    val = val
            elif (val.ndim == 2) or (val.ndim == 3 and val.shape[2] == 1):
                val = np.tile(np.atleast_3d(val), (1, 1, 3))
            elif val.ndim == 3 and val.shape[2] == 4:
                if self._visualization_mode == 'BGR':
                    val = val[:, :, [2, 1, 0]]
                else:
                    val = val[:, :, [0, 1, 2]]
        except KeyboardInterrupt:
            raise
        except Exception as e:
            logger = LoggerRoot.get_base_logger(TensorflowBinding)
            logger.warning('Failed decoding debug image [%s, %s, %s]' % (width, height, color_channels))
            logger.warning('Error: %s' % e)
            val = None
        return val

    def _add_image_numpy(self, tag, step, img_data_np, max_keep_images=None):
        # type: (str, int, Union[None, np.ndarray, str], int) -> ()
        # only report images every specific interval
        if step % self.image_report_freq != 0:
            return None

        if img_data_np is None:
            return

        # noinspection PyProtectedMember
        title, series = self.tag_splitter(tag, num_split_parts=3, default_title='Images', logdir_header='title',
                                          auto_reduce_num_split=True,
                                          force_add_prefix=self._logger._get_tensorboard_series_prefix())
        step = self._fix_step_counter(title, series, step)

        # check if this is a local temp file
        if isinstance(img_data_np, str):
            self._logger.report_image(
                title=title,
                series=series,
                iteration=step,
                local_path=img_data_np,
                delete_after_upload=True,
                max_image_history=self.max_keep_images if max_keep_images is None else max_keep_images,
            )
            return

        if img_data_np.dtype != np.uint8:
            # assume scale 0-1
            img_data_np = (img_data_np * 255).astype(np.uint8)

        # if 3d, pack into one big image
        if img_data_np.ndim == 4:
            dims = img_data_np.shape
            stack_dim = int(np.sqrt(dims[0]))
            # noinspection PyArgumentList
            res = img_data_np.reshape(stack_dim, stack_dim, *dims[1:]).transpose((0, 2, 1, 3, 4))
            tile_size_h = res.shape[0] * res.shape[1]
            tile_size_w = res.shape[2] * res.shape[3]
            img_data_np = res.reshape(tile_size_h, tile_size_w, -1)

        self._logger.report_image(
            title=title,
            series=series,
            iteration=step,
            image=img_data_np,
            max_image_history=self.max_keep_images if max_keep_images is None else max_keep_images,
        )

    def _add_image(self, tag, step, img_data):
        # only report images every specific interval
        if step % self.image_report_freq != 0:
            return None

        width = img_data.get('width')
        height = img_data.get('height')
        colorspace = img_data.get('colorspace')
        img_str = img_data['encodedImageString']
        matrix = self._decode_image(img_str, width=width, height=height, color_channels=colorspace)
        if matrix is None:
            return

        return self._add_image_numpy(tag=tag, step=step, img_data_np=matrix)

    def _add_scalar(self, tag, step, scalar_data):
        default_title = tag if not self._logger._get_tensorboard_auto_group_scalars() else 'Scalars'
        series_per_graph = self._logger._get_tensorboard_single_series_per_graph()

        # noinspection PyProtectedMember
        title, series = self.tag_splitter(
            tag, num_split_parts=1, default_title=default_title,
            logdir_header='title' if series_per_graph else 'series_last',
            force_add_prefix=self._logger._get_tensorboard_series_prefix()
        )

        step = self._fix_step_counter(title, series, step)
        tag = self._get_add_scalars_event_tag(default_title)

        possible_title = tag if series_per_graph else None
        possible_tag = None if series_per_graph else tag

        title = title + possible_title if possible_title else title
        series = possible_tag or series
        # update scalar cache
        num, value = self._scalar_report_cache.get((title, series), (0, 0))
        # nan outputs is a string, it's probably a NaN
        if isinstance(scalar_data, six.string_types):
            # noinspection PyBroadException
            try:
                scalar_data = float(scalar_data)
            except Exception:
                scalar_data = float('nan')
        # nan outputs nan
        self._scalar_report_cache[(title, series)] = \
            (num + 1,
             (value + scalar_data) if scalar_data == scalar_data else scalar_data)

        # only report images every specific interval
        if step % self.report_freq != 0:
            return None

        # calculate mean and zero cache
        num, value = self._scalar_report_cache.get((title, series), (0, 0))
        scalar_data = value / num
        self._scalar_report_cache[(title, series)] = (0, 0)

        self._logger.report_scalar(
            title=title,
            series=series,
            iteration=step,
            value=scalar_data,
        )

    def _add_histogram(self, tag, step, hist_data):
        # noinspection PyProtectedMember
        title, series = self.tag_splitter(tag, num_split_parts=1, default_title='Histograms',
                                          logdir_header='series',
                                          force_add_prefix=self._logger._get_tensorboard_series_prefix())

        self._grad_helper.add_histogram(
            title=title,
            series=series,
            step=step,
            hist_data=hist_data
        )

    def _add_plot(self, tag, step, values, vdict):
        # noinspection PyBroadException
        try:
            if values.get('floatVal'):
                plot_values = np.array(values.get('floatVal'), dtype=np.float32)
            else:
                plot_values = np.frombuffer(base64.b64decode(values['tensorContent'].encode('utf-8')),
                                            dtype=np.float32)
            plot_values = plot_values.reshape((int(values['tensorShape']['dim'][0]['size']),
                                               int(values['tensorShape']['dim'][1]['size'])))
            if 'metadata' in vdict:
                if tag not in self._series_name_lookup:
                    self._series_name_lookup[tag] = [(tag, vdict['metadata'].get('displayName', ''),
                                                      vdict['metadata']['pluginData']['pluginName'])]
                else:
                    # this should not happen, maybe it's another run, let increase the value
                    self._series_name_lookup[tag] += [(tag + '_%d' % (len(self._series_name_lookup[tag]) + 1),
                                                       vdict['metadata'].get('displayName', ''),
                                                       vdict['metadata']['pluginData']['pluginName'])]

            tag, series, plugin_name = self._series_name_lookup.get(tag, [(tag, tag, '')])[-1]

            if 'pr_curve' in plugin_name:
                # our thresholds are evenly distributed, in that
                #   width = 1.0 / (num_thresholds - 1)
                #   thresholds = [0.0, 1*width, 2*width, 3*width, ..., 1.0]
                num_thresholds = plot_values.shape[1]
                width = 1.0 / num_thresholds
                thresholds = np.arange(0.0, 1.0, width, dtype=plot_values.dtype)
                data_points = ['Threshold ', 'TP     ', 'FP     ', 'TN     ', 'FN     ', 'Precision ', ' Recall']
                series = [{'name': series, 'data': np.vstack((plot_values[-1], plot_values[-2])).T,
                           'labels': [''.join(data_points) + '<br> {:.3f}  '.format(thresholds[j]) +
                                      '  '.join(['%-3.2f' % v for v in plot_values[:, j]]) for j in
                                      range(num_thresholds)]}]
                reverse_xaxis = False
            else:
                reverse_xaxis = False
                series = [{'name': series, 'data': plot_values}]
            self._logger.report_line_plot(title=tag, series=series, xaxis='', yaxis='',
                                          iteration=step, reverse_xaxis=reverse_xaxis)
        except Exception:
            pass

    def _add_audio(self, tag, step, values, audio_data=None):
        # only report images every specific interval
        if step % self.image_report_freq != 0:
            return None

        if values:
            audio_str = values['encodedAudioString']
            audio_data = base64.b64decode(audio_str)
        if audio_data is None:
            return

        # noinspection PyProtectedMember
        title, series = self.tag_splitter(tag, num_split_parts=3, default_title='Audio', logdir_header='title',
                                          auto_reduce_num_split=True,
                                          force_add_prefix=self._logger._get_tensorboard_series_prefix())
        step = self._fix_step_counter(title, series, step)

        stream = BytesIO(audio_data)
        if values:
            file_extension = guess_extension(values['contentType']) or \
                '.{}'.format(values['contentType'].split('/')[-1])
        else:
            # assume wav as default
            file_extension = '.wav'
        self._logger.report_media(
            title=title,
            series=series,
            iteration=step,
            stream=stream,
            file_extension=file_extension,
            max_history=self.max_keep_images,
        )

    def _add_text(self, tag, step, tensor_bytes):
        # noinspection PyProtectedMember
        title, series = self.tag_splitter(tag, num_split_parts=3, default_title='Text', logdir_header='title',
                                          auto_reduce_num_split=True,
                                          force_add_prefix=self._logger._get_tensorboard_series_prefix())
        step = self._fix_step_counter(title, series, step)

        text = tensor_bytes.decode('utf-8', errors='replace')
        self._logger.report_media(
            title=title,
            series=series,
            iteration=step,
            stream=six.StringIO(text),
            file_extension='.txt',
            max_history=self.max_keep_images,
        )

    @staticmethod
    def _fix_step_counter(title, series, step):
        key = (title, series)
        if key not in EventTrainsWriter._title_series_wraparound_counter:
            EventTrainsWriter._title_series_wraparound_counter[key] = {'first_step': step, 'last_step': step,
                                                                       'adjust_counter': 0}
            return step
        wraparound_counter = EventTrainsWriter._title_series_wraparound_counter[key]
        # we decide on wrap around if the current step is less than 10% of the previous step
        # notice since counter is int and we want to avoid rounding error, we have double check in the if
        if step < wraparound_counter['last_step'] and step < 0.9 * wraparound_counter['last_step']:
            # adjust step base line
            wraparound_counter['adjust_counter'] += wraparound_counter['last_step'] + (1 if step <= 0 else step)

        # return adjusted step
        wraparound_counter['last_step'] = step
        return step + wraparound_counter['adjust_counter']

    def add_event(self, event, step=None, walltime=None, **_):
        supported_metrics = {
            'simpleValue', 'image', 'histo', 'tensor', 'audio'
        }

        def get_data(value_dict, metric_search_order):
            data = None
            metric_type = 'Unsupported'
            for variant in metric_search_order:
                data = value_dict.get(variant)
                if data is not None:
                    metric_type = variant
                    break
            return metric_type, data

        # Support multiple threads accessing this instance (i.e. let TF/Keras do what they need)
        with self._add_lock:
            # TODO: add report frequency threshold (i.e. if we are sending too much data, increase the report_freq)
            # we should measure reports per second and throttle back the reporting details accordingly
            msg_dict = MessageToDict(event)
            summary = msg_dict.get('summary')
            if summary is None:
                msg_dict.pop('step', None)
                msg_dict.pop('wallTime', None)
                keys_list = [key for key in msg_dict.keys() if len(key) > 0]
                keys_list = ', '.join(keys_list)
                LoggerRoot.get_base_logger(TensorflowBinding).debug(
                    'event summary not found, message type unsupported: %s' % keys_list)
                return
            value_dicts = summary.get('value')
            # noinspection PyUnusedLocal
            walltime = walltime or msg_dict.get('step')
            step = step or msg_dict.get('step')
            if step is None:
                # when we start a new epoch there is no step in the msg_dict,
                # we have to extract it manually
                if hasattr(event, 'step'):
                    step = int(event.step)
                else:
                    step = 0
                    LoggerRoot.get_base_logger(TensorflowBinding).debug(
                        'Received event without step, assuming step = {}'.format(step))
            else:
                step = int(step)
            self._max_step = max(self._max_step, step)
            if value_dicts is None:
                LoggerRoot.get_base_logger(TensorflowBinding).debug("Summary arrived without 'value'")
                return

            for vdict in value_dicts:
                tag = vdict.pop('tag', None)
                if tag is None:
                    # we should not get here
                    LoggerRoot.get_base_logger(TensorflowBinding).debug(
                        'No tag for \'value\' existing keys %s' % ', '.join(vdict.keys()))
                    continue
                metric, values = get_data(vdict, supported_metrics)
                if metric == 'simpleValue':
                    self._add_scalar(tag=tag, step=step, scalar_data=values)
                elif metric == 'histo':
                    self._add_histogram(tag=tag, step=step, hist_data=values)
                elif metric == 'image':
                    self._add_image(tag=tag, step=step, img_data=values)
                elif metric == 'audio':
                    self._add_audio(tag, step, values)
                elif metric == 'tensor' and values.get('dtype') == 'DT_STRING':
                    # generic tensor
                    tensor_bytes = base64.b64decode('\n'.join(values['stringVal']))
                    plugin_type = self._generic_tensor_type_name_lookup.get(tag) or \
                        vdict.get('metadata', {}).get('pluginData', {}).get('pluginName', '').lower()
                    if plugin_type == 'audio':
                        self._generic_tensor_type_name_lookup[tag] = plugin_type
                        self._add_audio(tag, step, None, tensor_bytes)
                    elif plugin_type == 'text':
                        self._generic_tensor_type_name_lookup[tag] = plugin_type
                        self._add_text(tag, step, tensor_bytes)
                    else:
                        # we do not support it
                        pass
                elif metric == 'tensor' and values.get('dtype') == 'DT_FLOAT':
                    self._add_plot(tag, step, values, vdict)
                else:
                    LoggerRoot.get_base_logger(TensorflowBinding).debug(
                        'Event unsupported. tag = %s, vdict keys [%s]' % (tag, ', '.join(vdict.keys())))
                    continue

    def get_logdir(self):
        """ Returns a temporary directory name for compatibility with FileWriter. This directory is not actually used.
        :return: '.'
        """
        return '.'

    def flush(self):
        """Flushes the event file to disk.

        Call this method to make sure that all pending events have been written to
        disk.
        """
        self._logger.flush()

    def close(self):
        """Flushes the event file to disk and close the file.

        Call this method when you do not need the summary writer anymore.
        """
        self._logger.flush()

    def reopen(self):
        """Reopens the EventFileWriter.

        Can be called after `close` to add more events in the same directory.
        The events will go into a new events file.

        Does nothing if the EventFileWriter was not closed.
        """
        pass

    def _get_add_scalars_event_tag(self, title_prefix):
        """

        :param str title_prefix: the table title prefix that was added to the series.
        :return: str same as tensorboard use
        """
        # HACK - this is tensorboard Summary util function, original path:
        # ~/torch/utils/tensorboard/summary.py
        def _clean_tag(name):
            import re as _re
            # noinspection RegExpRedundantEscape
            _INVALID_TAG_CHARACTERS = _re.compile(r'[^-/\w\.]')
            if name is not None:
                new_name = _INVALID_TAG_CHARACTERS.sub('_', name)
                new_name = new_name.lstrip('/')  # Remove leading slashes
                if new_name != name:
                    LoggerRoot.get_base_logger(TensorflowBinding).debug(
                        'Summary name %s is illegal; using %s instead.' % (name, new_name))
                    name = new_name
            return name

        main_path = self._logdir
        # noinspection PyBroadException
        try:
            main_path = _clean_tag(main_path)
            origin_tag = main_path.rpartition("/")[2].replace(title_prefix, "", 1)
            if title_prefix and origin_tag[0] == "_":  # add_scalars tag
                origin_tag = origin_tag[1:]  # Remove the first "_" that was added by the main_tag in tensorboard
            else:
                return ""
        except Exception:
            origin_tag = ""
        return origin_tag

    @classmethod
    def update_current_task(cls, task):
        if cls.__main_task != task:
            with cls._add_lock:
                cls._series_name_lookup = {}
                cls._title_series_writers_lookup = {}
                cls._event_writers_id_to_logdir = {}
                cls._title_series_wraparound_counter = {}
        cls.__main_task = task


# noinspection PyCallingNonCallable
class ProxyEventsWriter(object):
    def __init__(self, events):
        IsTensorboardInit.set_tensorboard_used()
        self._events = events

    def _get_sentinel_event(self):
        ret = None
        for ev in self._events:
            if hasattr(ev, '_get_sentinel_event'):
                # noinspection PyProtectedMember
                ret = ev._get_sentinel_event()
        return ret

    def get_logdir(self):
        ret = None
        for ev in self._events:
            if hasattr(ev, 'get_logdir'):
                ret = ev.get_logdir()
        return ret

    def reopen(self):
        ret = None
        for ev in self._events:
            if hasattr(ev, 'reopen'):
                ret = ev.reopen()
        return ret

    def add_event(self, *args, **kwargs):
        ret = None
        for ev in self._events:
            if hasattr(ev, 'add_event'):
                ret = ev.add_event(*args, **kwargs)
        return ret

    def flush(self):
        ret = None
        for ev in self._events:
            if hasattr(ev, 'flush'):
                ret = ev.flush()
        return ret

    def close(self):
        ret = None
        for ev in self._events:
            if hasattr(ev, 'close'):
                ret = ev.close()
        return ret


# noinspection PyPep8Naming
class PatchSummaryToEventTransformer(object):
    __main_task = None
    __original_getattribute = None
    __original_getattributeX = None
    _original_add_event = None
    _original_add_eventT = None
    _original_add_eventX = None
    defaults_dict = dict(
        report_freq=1, image_report_freq=1, histogram_update_freq_multiplier=5,
        histogram_granularity=50)

    @staticmethod
    def trains_object(self):
        if isinstance(self.event_writer, ProxyEventsWriter):
            # noinspection PyProtectedMember
            trains_writer = [e for e in self.event_writer._events if isinstance(e, EventTrainsWriter)]
            return trains_writer[0] if trains_writer else None
        elif isinstance(self.event_writer, EventTrainsWriter):
            return self.event_writer
        if not self.__dict__.get('_trains_defaults'):
            self.__dict__['_trains_defaults'] = {}
        return self.__dict__['_trains_defaults']

    @staticmethod
    def update_current_task(task, **kwargs):
        PatchSummaryToEventTransformer.defaults_dict.update(kwargs)
        PatchSummaryToEventTransformer.__main_task = task
        # make sure we patched the SummaryToEventTransformer
        PatchSummaryToEventTransformer._patch_summary_to_event_transformer()
        PostImportHookPatching.add_on_import('tensorflow',
                                             PatchSummaryToEventTransformer._patch_summary_to_event_transformer)
        PostImportHookPatching.add_on_import('torch',
                                             PatchSummaryToEventTransformer._patch_summary_to_event_transformer)
        PostImportHookPatching.add_on_import('tensorboardX',
                                             PatchSummaryToEventTransformer._patch_summary_to_event_transformer)

    @staticmethod
    def _patch_summary_to_event_transformer():
        if 'tensorflow' in sys.modules:
            try:
                from tensorflow.python.summary.writer.writer import SummaryToEventTransformer  # noqa
                # only patch once
                if PatchSummaryToEventTransformer.__original_getattribute is None:
                    PatchSummaryToEventTransformer.__original_getattribute = SummaryToEventTransformer.__getattribute__
                    SummaryToEventTransformer.__getattribute__ = PatchSummaryToEventTransformer._patched_getattribute
                    setattr(SummaryToEventTransformer, 'clearml',
                            property(PatchSummaryToEventTransformer.trains_object))
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).debug(str(ex))

        if 'torch' in sys.modules:
            try:
                # only patch once
                if PatchSummaryToEventTransformer._original_add_eventT is None:
                    # noinspection PyUnresolvedReferences
                    from torch.utils.tensorboard.writer import FileWriter as FileWriterT  # noqa
                    PatchSummaryToEventTransformer._original_add_eventT = FileWriterT.add_event
                    FileWriterT.add_event = PatchSummaryToEventTransformer._patched_add_eventT
                    setattr(FileWriterT, 'clearml', None)
            except ImportError:
                # this is a new version of TensorflowX
                pass
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).debug(str(ex))

        if 'tensorboardX' in sys.modules:
            try:
                # only patch once
                if PatchSummaryToEventTransformer.__original_getattributeX is None:
                    # noinspection PyUnresolvedReferences
                    from tensorboardX.writer import SummaryToEventTransformer as SummaryToEventTransformerX  # noqa
                    PatchSummaryToEventTransformer.__original_getattributeX = \
                        SummaryToEventTransformerX.__getattribute__
                    SummaryToEventTransformerX.__getattribute__ = PatchSummaryToEventTransformer._patched_getattributeX
                    setattr(SummaryToEventTransformerX, 'clearml',
                            property(PatchSummaryToEventTransformer.trains_object))
            except ImportError:
                # this is a new version of TensorflowX
                pass
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).debug(str(ex))

            if PatchSummaryToEventTransformer.__original_getattributeX is None:
                try:
                    # only patch once
                    if PatchSummaryToEventTransformer._original_add_eventX is None:
                        from tensorboardX.writer import FileWriter as FileWriterX  # noqa
                        PatchSummaryToEventTransformer._original_add_eventX = FileWriterX.add_event
                        FileWriterX.add_event = PatchSummaryToEventTransformer._patched_add_eventX
                        setattr(FileWriterX, 'clearml', None)
                except ImportError:
                    # this is a new version of TensorflowX
                    pass
                except Exception as ex:
                    LoggerRoot.get_base_logger(TensorflowBinding).debug(str(ex))

    @staticmethod
    def _patched_add_eventT(self, *args, **kwargs):
        if not hasattr(self, 'clearml') or not PatchSummaryToEventTransformer.__main_task:
            return PatchSummaryToEventTransformer._original_add_eventT(self, *args, **kwargs)
        if not self.clearml:  # noqa
            # noinspection PyBroadException
            try:
                logdir = self.get_logdir()
            except Exception:
                logdir = None
            self.clearml = EventTrainsWriter(PatchSummaryToEventTransformer.__main_task.get_logger(),
                                             logdir=logdir, **PatchSummaryToEventTransformer.defaults_dict)
        # noinspection PyBroadException
        try:
            self.clearml.add_event(*args, **kwargs)
        except Exception:
            pass
        return PatchSummaryToEventTransformer._original_add_eventT(self, *args, **kwargs)

    @staticmethod
    def _patched_add_eventX(self, *args, **kwargs):
        if not hasattr(self, 'clearml') or not PatchSummaryToEventTransformer.__main_task:
            return PatchSummaryToEventTransformer._original_add_eventX(self, *args, **kwargs)
        if not self.clearml:
            # noinspection PyBroadException
            try:
                logdir = self.get_logdir()
            except Exception:
                logdir = None
            self.clearml = EventTrainsWriter(PatchSummaryToEventTransformer.__main_task.get_logger(),
                                             logdir=logdir, **PatchSummaryToEventTransformer.defaults_dict)
        # noinspection PyBroadException
        try:
            self.clearml.add_event(*args, **kwargs)
        except Exception:
            pass
        return PatchSummaryToEventTransformer._original_add_eventX(self, *args, **kwargs)

    @staticmethod
    def _patched_getattribute(self, attr):
        get_base = PatchSummaryToEventTransformer.__original_getattribute
        return PatchSummaryToEventTransformer._patched_getattribute_(self, attr, get_base)

    @staticmethod
    def _patched_getattributeX(self, attr):
        get_base = PatchSummaryToEventTransformer.__original_getattributeX
        return PatchSummaryToEventTransformer._patched_getattribute_(self, attr, get_base)

    @staticmethod
    def _patched_getattribute_(self, attr, get_base):
        # no main task, zero chance we have an ClearML event logger
        if PatchSummaryToEventTransformer.__main_task is None:
            return get_base(self, attr)

        # check if we already have an ClearML event logger
        __dict__ = get_base(self, '__dict__')
        if 'event_writer' not in __dict__ or \
                isinstance(__dict__['event_writer'], (ProxyEventsWriter, EventTrainsWriter)):
            return get_base(self, attr)

        # patch the events writer field, and add a double Event Logger (ClearML and original)
        base_eventwriter = __dict__['event_writer']
        # noinspection PyBroadException
        try:
            logdir = base_eventwriter.get_logdir()
        except Exception:
            logdir = None
        defaults_dict = __dict__.get('_trains_defaults') or PatchSummaryToEventTransformer.defaults_dict
        trains_event = EventTrainsWriter(PatchSummaryToEventTransformer.__main_task.get_logger(),
                                         logdir=logdir, **defaults_dict)

        # order is important, the return value of ProxyEventsWriter is the last object in the list
        __dict__['event_writer'] = ProxyEventsWriter([trains_event, base_eventwriter])
        return get_base(self, attr)


class _ModelAdapter(object):
    """ Model adapter which extends the save and save_weights methods of a Keras Model instance """
    _model = None  # type: Any
    _output_model = None  # type: OutputModel

    def __init__(self, model, output_model):
        super(_ModelAdapter, self).__init__()
        super(_ModelAdapter, self).__setattr__('_model', model)
        super(_ModelAdapter, self).__setattr__('_output_model', output_model)
        super(_ModelAdapter, self).__setattr__('_logger', LoggerRoot.get_base_logger(TensorflowBinding))

    def __getattr__(self, attr):
        return getattr(self._model, attr)

    def __setattr__(self, key, value):
        return setattr(self._model, key, value)

    def save(self, filepath, overwrite=True, include_optimizer=True):
        self._model.save(filepath=filepath, overwrite=overwrite, include_optimizer=include_optimizer)
        # TODO: auto generate new objects of filename changes
        try:
            self._output_model.update_weights(weights_filename=filepath, auto_delete_file=True)
        except Exception as ex:
            self._logger.error(str(ex))

    def save_weights(self, filepath, overwrite=True):
        self._model.save_weights(filepath=filepath, overwrite=overwrite)
        # TODO: auto generate new objects of filename changes
        try:
            self._output_model.update_weights(weights_filename=filepath, auto_delete_file=True)
        except Exception as ex:
            self._logger.error(str(ex))


class PatchModelCheckPointCallback(object):
    __main_task = None
    __original_getattribute = None
    defaults_dict = dict(
        config_text=None,
        config_dict=None,
        label_enumeration=None,
        name=None,
        comment=None)

    @staticmethod
    def trains_object(self):
        if isinstance(self.model, _ModelAdapter):
            # noinspection PyProtectedMember
            return self.model._output_model
        if not self.__dict__.get('_trains_defaults'):
            self.__dict__['_trains_defaults'] = {}
        return self.__dict__['_trains_defaults']

    @staticmethod
    def update_current_task(task, **kwargs):
        PatchModelCheckPointCallback.defaults_dict.update(kwargs)
        PatchModelCheckPointCallback.__main_task = task
        # make sure we patched the SummaryToEventTransformer
        PatchModelCheckPointCallback._patch_model_checkpoint()
        PostImportHookPatching.add_on_import('keras', PatchModelCheckPointCallback._patch_model_checkpoint)
        PostImportHookPatching.add_on_import('tensorflow', PatchModelCheckPointCallback._patch_model_checkpoint)

    @staticmethod
    def _patch_model_checkpoint():
        is_keras = 'keras' in sys.modules
        is_tf_keras = 'tensorflow' in sys.modules
        callbacks = None
        if is_keras:
            try:
                import keras.callbacks as callbacks  # noqa
            except ImportError:
                is_keras = False
        if not is_keras and is_tf_keras:
            try:
                # hack: make sure tensorflow.__init__ is called
                import tensorflow  # noqa
                import tensorflow.python.keras.callbacks as callbacks  # noqa
            except ImportError:
                is_tf_keras = False
                callbacks = None
        # we have nothing, quit
        if not is_keras and not is_tf_keras:
            return

        try:
            # only patch once
            if PatchModelCheckPointCallback.__original_getattribute is None and callbacks is not None:
                PatchModelCheckPointCallback.__original_getattribute = callbacks.ModelCheckpoint.__getattribute__
                callbacks.ModelCheckpoint.__getattribute__ = PatchModelCheckPointCallback._patched_getattribute
                setattr(callbacks.ModelCheckpoint, 'clearml',
                        property(PatchModelCheckPointCallback.trains_object))

        except Exception as ex:
            LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))

    @staticmethod
    def _patched_getattribute(self, attr):
        get_base = PatchModelCheckPointCallback.__original_getattribute

        # no main task, zero chance we have an ClearML event logger
        if PatchModelCheckPointCallback.__main_task is None:
            return get_base(self, attr)

        # check if we already have an ClearML event logger
        __dict__ = get_base(self, '__dict__')
        if 'model' not in __dict__ or \
                isinstance(__dict__['model'], _ModelAdapter):
            return get_base(self, attr)

        # patch the events writer field, and add a double Event Logger (ClearML and original)
        base_model = __dict__['model']
        defaults_dict = __dict__.get('_trains_defaults') or PatchModelCheckPointCallback.defaults_dict
        output_model = OutputModel(
            PatchModelCheckPointCallback.__main_task,
            config_text=defaults_dict.get('config_text'),
            config_dict=defaults_dict.get('config_dict'),
            name=defaults_dict.get('name'),
            comment=defaults_dict.get('comment'),
            label_enumeration=defaults_dict.get('label_enumeration') or
            PatchModelCheckPointCallback.__main_task.get_labels_enumeration(),
            framework=Framework.keras,
        )
        output_model.set_upload_destination(
            PatchModelCheckPointCallback.__main_task.get_output_destination(raise_on_error=False))
        trains_model = _ModelAdapter(base_model, output_model)

        # order is important, the return value of ProxyEventsWriter is the last object in the list
        __dict__['model'] = trains_model
        return get_base(self, attr)


# noinspection PyProtectedMember,PyUnresolvedReferences
class PatchTensorFlowEager(object):
    __main_task = None
    __original_fn_scalar = None
    __original_fn_hist = None
    __original_fn_image = None
    __trains_event_writer = {}
    __tf_tb_writer_id_to_logdir = {}
    defaults_dict = dict(
        report_freq=1, image_report_freq=1, histogram_update_freq_multiplier=5,
        histogram_granularity=50)

    @staticmethod
    def update_current_task(task, **kwargs):
        if task != PatchTensorFlowEager.__main_task:
            PatchTensorFlowEager.__trains_event_writer = {}

        PatchTensorFlowEager.defaults_dict.update(kwargs)
        PatchTensorFlowEager.__main_task = task
        # make sure we patched the SummaryToEventTransformer
        PatchTensorFlowEager._patch_summary_ops()
        PostImportHookPatching.add_on_import('tensorflow', PatchTensorFlowEager._patch_summary_ops)

    @staticmethod
    def _patch_summary_ops():
        if PatchTensorFlowEager.__original_fn_scalar is not None:
            return
        if 'tensorflow' in sys.modules:
            try:
                # hack: make sure tensorflow.__init__ is called
                import tensorflow  # noqa
                from tensorflow.python.ops import gen_summary_ops  # noqa
                PatchTensorFlowEager.__original_fn_scalar = gen_summary_ops.write_scalar_summary
                gen_summary_ops.write_scalar_summary = PatchTensorFlowEager._write_scalar_summary
                PatchTensorFlowEager.__original_fn_image = gen_summary_ops.write_image_summary
                gen_summary_ops.write_image_summary = PatchTensorFlowEager._write_image_summary
                PatchTensorFlowEager.__original_fn_hist = gen_summary_ops.write_histogram_summary
                gen_summary_ops.write_histogram_summary = PatchTensorFlowEager._write_hist_summary
                PatchTensorFlowEager.__write_summary = gen_summary_ops.write_summary
                gen_summary_ops.write_summary = PatchTensorFlowEager._write_summary
                gen_summary_ops.create_summary_file_writer = partial(IsTensorboardInit._patched_tb__init__,
                                                                     gen_summary_ops.create_summary_file_writer)
                gen_summary_ops.create_summary_db_writer = partial(IsTensorboardInit._patched_tb__init__,
                                                                   gen_summary_ops.create_summary_db_writer)
            except ImportError:
                pass
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).debug(str(ex))

            # tensorflow 2.7 support (getting logdir)
            try:
                import tensorflow  # noqa
                import tensorflow.python  # noqa
                from tensorflow.python.ops import gen_summary_ops
                gen_summary_ops.create_summary_file_writer = _patched_call(
                    gen_summary_ops.create_summary_file_writer,
                    PatchTensorFlowEager._create_summary_file_writer
                )
            except Exception:
                pass

    @staticmethod
    def _create_summary_file_writer(original_fn, *args, **kwargs):
        # noinspection PyBroadException
        try:
            a_logdir = None
            a_writer = None
            if kwargs and 'logdir' in kwargs:
                a_logdir = kwargs.get('logdir')
            elif args and len(args) >= 2:
                a_logdir = args[1]

            if kwargs and 'writer' in kwargs:
                a_writer = kwargs.get('writer')
            elif args and len(args) >= 1:
                a_writer = args[0]

            if a_writer is not None and a_logdir is not None:
                a_logdir = a_logdir.numpy().decode()
                PatchTensorFlowEager.__tf_tb_writer_id_to_logdir[id(a_writer)] = a_logdir
        except Exception:
            pass

        return original_fn(*args, **kwargs)

    @staticmethod
    def _get_event_writer(writer):
        if not PatchTensorFlowEager.__main_task:
            return None
        if not PatchTensorFlowEager.__trains_event_writer.get(id(writer)):
            # noinspection PyBroadException
            try:
                logdir = writer.get_logdir()
            except Exception:
                # check if we are in eager mode, let's get the global context lopdir
                # noinspection PyBroadException
                try:
                    from tensorflow.python.eager import context  # noqa
                    logdir = context.context().summary_writer._init_op_fn.keywords.get('logdir')
                except Exception:
                    # noinspection PyBroadException
                    try:
                        from tensorflow.python.ops.summary_ops_v2 import _summary_state  # noqa
                        logdir = _summary_state.writer._init_op_fn.keywords.get('logdir')
                    except Exception:
                        try:
                            logdir = PatchTensorFlowEager.__tf_tb_writer_id_to_logdir[id(writer)]
                        except Exception:
                            logdir = None
                # noinspection PyBroadException
                try:
                    if logdir is not None:
                        logdir = logdir.numpy().decode() if not isinstance(logdir, str) else logdir
                except Exception:
                    logdir = None

            PatchTensorFlowEager.__trains_event_writer[id(writer)] = EventTrainsWriter(
                logger=PatchTensorFlowEager.__main_task.get_logger(), logdir=logdir,
                **PatchTensorFlowEager.defaults_dict)
        return PatchTensorFlowEager.__trains_event_writer[id(writer)]

    @staticmethod
    def trains_object(self):
        if not PatchTensorFlowEager.__trains_event_writer:
            return None
        return PatchTensorFlowEager.__trains_event_writer.get(
            id(self), list(PatchTensorFlowEager.__trains_event_writer.values())[0])

    @staticmethod
    def _write_summary(writer, step, tensor, tag, summary_metadata, name=None, **kwargs):
        event_writer = PatchTensorFlowEager._get_event_writer(writer)
        # make sure we can get the tensors values
        if event_writer and isinstance(step, int) or hasattr(step, 'numpy'):
            # noinspection PyBroadException
            try:
                plugin_type = summary_metadata.decode()
                # remove any none alpha numeric value
                plugin_type = plugin_type[next(i for i, c in enumerate(plugin_type) if c >= 'A'):]
                if plugin_type.startswith('scalars'):
                    event_writer._add_scalar(tag=str(tag),
                                             step=int(step.numpy()) if not isinstance(step, int) else step,
                                             scalar_data=tensor.numpy())
                elif plugin_type.startswith('images'):
                    img_data_np = tensor.numpy()
                    PatchTensorFlowEager._add_image_event_helper(event_writer, img_data_np=img_data_np,
                                                                 tag=tag, step=step, **kwargs)
                elif plugin_type.startswith('histograms'):
                    event_writer._add_histogram(
                        tag=str(tag), step=int(step.numpy()) if not isinstance(step, int) else step,
                        hist_data=tensor.numpy()
                    )
                elif plugin_type.startswith('text'):
                    event_writer._add_text(
                        tag=str(tag), step=int(step.numpy()) if not isinstance(step, int) else step,
                        tensor_bytes=tensor.numpy()
                    )
                elif 'audio' in plugin_type:
                    audio_bytes_list = [a for a in tensor.numpy().flatten() if a]
                    for i, audio_bytes in enumerate(audio_bytes_list):
                        event_writer._add_audio(tag=str(tag) + ('/{}'.format(i) if len(audio_bytes_list) > 1 else ''),
                                                step=int(step.numpy()) if not isinstance(step, int) else step,
                                                values=None, audio_data=audio_bytes)
                else:
                    pass  # print('unsupported plugin_type', plugin_type)
            except Exception:
                pass
        return PatchTensorFlowEager.__write_summary(writer, step, tensor, tag, summary_metadata, name, **kwargs)

    @staticmethod
    def _write_scalar_summary(writer, step, tag, value, name=None, **kwargs):
        event_writer = PatchTensorFlowEager._get_event_writer(writer)
        if event_writer and isinstance(step, int) or hasattr(step, 'numpy'):
            try:
                event_writer._add_scalar(tag=str(tag),
                                         step=int(step.numpy()) if not isinstance(step, int) else step,
                                         scalar_data=value.numpy())
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))
        elif event_writer:
            def _report_summary_op(a_writer, a_step, a_tag, a_value, a_name=None, **_):
                if isinstance(a_step, int) or hasattr(a_step, 'numpy'):
                    try:
                        str_tag = a_tag.numpy()
                        str_tag = str_tag.decode() if isinstance(str_tag, bytes) else str(str_tag)
                        event_writer._add_scalar(
                            tag=str_tag,
                            step=int(a_step.numpy()) if not isinstance(a_step, int) else a_step,
                            scalar_data=a_value.numpy())
                    except Exception as a_ex:
                        LoggerRoot.get_base_logger(TensorflowBinding).warning(
                            '_report_summary_op: {}'.format(str(a_ex)))

            # this is a mix of eager and graph execution
            try:
                from tensorflow.python.eager import context as _context
                if not _context.executing_eagerly():
                    from tensorflow import py_function
                    # just creating the operator is enough (for some reason)
                    # to make sure it is added into the execution tree.
                    # the operator itself, will do the reporting to the backend
                    py_function(
                        _report_summary_op,
                        inp=[writer, step, tag, value, name], Tout=[])
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))

        return PatchTensorFlowEager.__original_fn_scalar(writer, step, tag, value, name, **kwargs)

    @staticmethod
    def _write_hist_summary(writer, step, tag, values, name, **kwargs):
        event_writer = PatchTensorFlowEager._get_event_writer(writer)
        if event_writer and isinstance(step, int) or hasattr(step, 'numpy'):
            try:
                event_writer._add_histogram(
                    tag=str(tag), step=int(step.numpy()) if not isinstance(step, int) else step,
                    hist_data=values.numpy()
                )
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))
        elif event_writer:
            def _report_summary_op(a_writer, a_step, a_tag, a_value, a_name=None, **_):
                if isinstance(a_step, int) or hasattr(a_step, 'numpy'):
                    try:
                        str_tag = a_tag.numpy()
                        str_tag = str_tag.decode() if isinstance(str_tag, bytes) else str(str_tag)
                        event_writer._add_histogram(
                            tag=str_tag,
                            step=int(a_step.numpy()) if not isinstance(a_step, int) else a_step,
                            hist_data=a_value.numpy()
                        )
                    except Exception as a_ex:
                        LoggerRoot.get_base_logger(TensorflowBinding).warning(
                            '_report_summary_op: {}'.format(str(a_ex)))

            # this is a mix of eager and graph execution
            try:
                from tensorflow.python.eager import context as _context
                if not _context.executing_eagerly():
                    from tensorflow import py_function
                    # just creating the operator is enough (for some reason)
                    # to make sure it is added into the execution tree.
                    # the operator itself, will do the reporting to the backend
                    py_function(
                        _report_summary_op,
                        inp=[writer, step, tag, values, name], Tout=[])
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))

        return PatchTensorFlowEager.__original_fn_hist(writer, step, tag, values, name, **kwargs)

    @staticmethod
    def _write_image_summary(writer, step, tag, tensor, bad_color, max_images, name, **kwargs):
        event_writer = PatchTensorFlowEager._get_event_writer(writer)
        if event_writer and isinstance(step, int) or hasattr(step, 'numpy'):
            try:
                PatchTensorFlowEager._add_image_event_helper(event_writer, img_data_np=tensor.numpy(),
                                                             tag=tag, step=step, **kwargs)
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))
        elif event_writer:
            def _report_summary_op(a_writer, a_step, a_tag, a_tensor, a_bad_color, a_max_images, a_name=None, **_):
                if isinstance(a_step, int) or hasattr(a_step, 'numpy'):
                    try:
                        str_tag = a_tag.numpy()
                        str_tag = str_tag.decode() if isinstance(str_tag, bytes) else str(str_tag)
                        PatchTensorFlowEager._add_image_event_helper(
                            event_writer, img_data_np=a_tensor.numpy(),
                            tag=str_tag, step=a_step, **kwargs)

                    except Exception as a_ex:
                        LoggerRoot.get_base_logger(TensorflowBinding).warning(
                            '_report_summary_op: {}'.format(str(a_ex)))

            # this is a mix of eager and graph execution
            try:
                from tensorflow.python.eager import context as _context
                if not _context.executing_eagerly():
                    from tensorflow import py_function
                    # just creating the operator is enough (for some reason)
                    # to make sure it is added into the execution tree.
                    # the operator itself, will do the reporting to the backend
                    py_function(
                        _report_summary_op,
                        inp=[writer, step, tag, tensor, bad_color, max_images, name], Tout=[])
            except Exception as ex:
                LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))

        return PatchTensorFlowEager.__original_fn_image(
            writer, step, tag, tensor, bad_color, max_images, name, **kwargs)

    @staticmethod
    def _add_image_event_helper(event_writer, img_data_np, tag, step, **kwargs):
        if img_data_np.ndim == 1 and img_data_np.size >= 3 and \
                (len(img_data_np[0]) < 10 and len(img_data_np[1]) < 10):
            # this is just for making sure these are actually valid numbers
            width = int(img_data_np[0].decode())  # noqa: F841
            height = int(img_data_np[1].decode())  # noqa: F841
            for i in range(2, img_data_np.size):
                img_data = {'width': None, 'height': None,
                            'colorspace': 'RGB', 'encodedImageString': img_data_np[i]}
                image_tag = str(tag) + '/sample_{}'.format(i - 2) if img_data_np.size > 3 else str(tag)
                event_writer._add_image(tag=image_tag,
                                        step=int(step.numpy()) if not isinstance(step, int) else step,
                                        img_data=img_data)
        else:
            event_writer._add_image_numpy(tag=str(tag),
                                          step=int(step.numpy()) if not isinstance(step, int) else step,
                                          img_data_np=img_data_np,
                                          max_keep_images=kwargs.get('max_images'))

    @staticmethod
    def _nothing_op(*_, **__):
        """Convenient else branch for when summaries do not record."""
        from tensorflow.python.framework import constant_op
        return constant_op.constant(False)


# noinspection PyPep8Naming,SpellCheckingInspection
class PatchKerasModelIO(object):
    __main_task = None
    __patched_keras = None
    __patched_tensorflow = None

    @staticmethod
    def update_current_task(task, **_):
        PatchKerasModelIO.__main_task = task
        PatchKerasModelIO._patch_model_checkpoint()
        PostImportHookPatching.add_on_import('tensorflow', PatchKerasModelIO._patch_model_checkpoint)
        PostImportHookPatching.add_on_import('keras', PatchKerasModelIO._patch_model_checkpoint)

    @staticmethod
    def _patch_model_checkpoint():
        if 'keras' in sys.modules and not PatchKerasModelIO.__patched_keras:
            try:
                from keras.engine.network import Network  # noqa
            except ImportError:
                Network = None
            try:
                from keras.engine.functional import Functional  # noqa
            except ImportError:
                Functional = None
            try:
                from keras.engine.sequential import Sequential  # noqa
            except ImportError:
                Sequential = None
            try:
                from keras import models as keras_saving  # noqa
            except ImportError:
                keras_saving = None
            # check that we are not patching anything twice
            if PatchKerasModelIO.__patched_tensorflow:
                PatchKerasModelIO.__patched_keras = [
                    Network if PatchKerasModelIO.__patched_tensorflow[0] != Network else None,
                    Sequential if PatchKerasModelIO.__patched_tensorflow[1] != Sequential else None,
                    keras_saving if PatchKerasModelIO.__patched_tensorflow[2] != keras_saving else None,
                    Functional if PatchKerasModelIO.__patched_tensorflow[3] != Functional else None,
                    None,
                    None,
                ]
            else:
                PatchKerasModelIO.__patched_keras = [Network, Sequential, keras_saving, Functional, None, None]
            PatchKerasModelIO._patch_io_calls(*PatchKerasModelIO.__patched_keras)

        if 'tensorflow' in sys.modules and not PatchKerasModelIO.__patched_tensorflow:
            try:
                # hack: make sure tensorflow.__init__ is called
                import tensorflow  # noqa
                from tensorflow.python.keras.engine.network import Network  # noqa
            except ImportError:
                Network = None
            try:
                # hack: make sure tensorflow.__init__ is called
                import tensorflow  # noqa
                from tensorflow.python.keras.engine.functional import Functional  # noqa
            except ImportError:
                Functional = None
            try:
                # hack: make sure tensorflow.__init__ is called
                import tensorflow  # noqa
                from tensorflow.python.keras.engine.sequential import Sequential  # noqa
            except ImportError:
                Sequential = None
            try:
                # hack: make sure tensorflow.__init__ is called
                import tensorflow  # noqa
                from tensorflow.python.keras import models as keras_saving_legacy  # noqa
            except ImportError:
                keras_saving_legacy = None

            try:
                # hack: make sure tensorflow.__init__ is called
                import tensorflow  # noqa
                from tensorflow.keras import models as keras_saving  # noqa
            except ImportError:
                keras_saving = None

            try:
                # hack: make sure tensorflow.__init__ is called
                import tensorflow  # noqa
                from tensorflow.python.keras.saving import hdf5_format as keras_hdf5  # noqa
            except ImportError:
                keras_hdf5 = None

            if PatchKerasModelIO.__patched_keras:
                PatchKerasModelIO.__patched_tensorflow = [
                    Network if PatchKerasModelIO.__patched_keras[0] != Network else None,
                    Sequential if PatchKerasModelIO.__patched_keras[1] != Sequential else None,
                    keras_saving if PatchKerasModelIO.__patched_keras[2] != keras_saving else None,
                    Functional if PatchKerasModelIO.__patched_keras[3] != Functional else None,
                    keras_saving_legacy if PatchKerasModelIO.__patched_keras[4] != keras_saving_legacy else None,
                    keras_hdf5 if PatchKerasModelIO.__patched_keras[5] != keras_hdf5 else None,
                ]
            else:
                PatchKerasModelIO.__patched_tensorflow = [
                    Network, Sequential, keras_saving, Functional, keras_saving_legacy, keras_hdf5]
            PatchKerasModelIO._patch_io_calls(*PatchKerasModelIO.__patched_tensorflow)

    @staticmethod
    def _patch_io_calls(Network, Sequential, keras_saving, Functional, keras_saving_legacy=None, keras_hdf5=None):
        try:
            if Sequential is not None:
                Sequential._updated_config = _patched_call(Sequential._updated_config,
                                                           PatchKerasModelIO._updated_config)
                if hasattr(Sequential.from_config, '__func__'):
                    # noinspection PyUnresolvedReferences
                    Sequential.from_config = classmethod(_patched_call(Sequential.from_config.__func__,
                                                                       PatchKerasModelIO._from_config))
                else:
                    Sequential.from_config = _patched_call(Sequential.from_config, PatchKerasModelIO._from_config)

            if Network is not None:
                Network._updated_config = _patched_call(Network._updated_config, PatchKerasModelIO._updated_config)
                if hasattr(Sequential.from_config, '__func__'):
                    # noinspection PyUnresolvedReferences
                    Network.from_config = classmethod(_patched_call(Network.from_config.__func__,
                                                                    PatchKerasModelIO._from_config))
                else:
                    Network.from_config = _patched_call(Network.from_config, PatchKerasModelIO._from_config)
                Network.save = _patched_call(Network.save, PatchKerasModelIO._save)
                Network.save_weights = _patched_call(Network.save_weights, PatchKerasModelIO._save_weights)
                Network.load_weights = _patched_call(Network.load_weights, PatchKerasModelIO._load_weights)
            elif Functional is not None:
                Functional._updated_config = _patched_call(
                    Functional._updated_config, PatchKerasModelIO._updated_config)
                if hasattr(Sequential.from_config, '__func__'):
                    # noinspection PyUnresolvedReferences
                    Functional.from_config = classmethod(_patched_call(Functional.from_config.__func__,
                                                                       PatchKerasModelIO._from_config))
                else:
                    Functional.from_config = _patched_call(Functional.from_config, PatchKerasModelIO._from_config)
                Functional.save = _patched_call(Functional.save, PatchKerasModelIO._save)
                Functional.save_weights = _patched_call(Functional.save_weights, PatchKerasModelIO._save_weights)
                Functional.load_weights = _patched_call(Functional.load_weights, PatchKerasModelIO._load_weights)

            if keras_saving is not None:
                keras_saving.save_model = _patched_call(keras_saving.save_model, PatchKerasModelIO._save_model)
                keras_saving.load_model = _patched_call(keras_saving.load_model, PatchKerasModelIO._load_model)

            if keras_saving_legacy is not None:
                keras_saving_legacy.save_model = _patched_call(
                    keras_saving_legacy.save_model, PatchKerasModelIO._save_model)
                keras_saving_legacy.load_model = _patched_call(
                    keras_saving_legacy.load_model, PatchKerasModelIO._load_model)

            if keras_hdf5 is not None:
                keras_hdf5.save_weights_to_hdf5_group = _patched_call(
                    keras_hdf5.save_weights_to_hdf5_group, PatchKerasModelIO._save_weights)
                keras_hdf5.load_weights_from_hdf5_group = _patched_call(
                    keras_hdf5.load_weights_from_hdf5_group, PatchKerasModelIO._load_weights)
                keras_hdf5.load_weights_from_hdf5_group_by_name = _patched_call(
                    keras_hdf5.load_weights_from_hdf5_group_by_name, PatchKerasModelIO._load_weights)
                if hasattr(keras_hdf5, 'load_model_from_hdf5'):
                    keras_hdf5.load_model_from_hdf5 = _patched_call(
                        keras_hdf5.load_model_from_hdf5, PatchKerasModelIO._load_model)
                if hasattr(keras_hdf5, 'save_model_to_hdf5'):
                    keras_hdf5.save_model_to_hdf5 = _patched_call(
                        keras_hdf5.save_model_to_hdf5, PatchKerasModelIO._save_model)

        except Exception as ex:
            LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))

    @staticmethod
    def _updated_config(original_fn, self):
        config = original_fn(self)
        # check if we have main task
        if PatchKerasModelIO.__main_task is None:
            return config

        try:
            # there is no actual file, so we create the OutputModel without one

            # check if object already has InputModel
            if not hasattr(self, 'trains_out_model'):
                self.trains_out_model = []

            # check if object already has InputModel
            model_name_id = config.get('name', getattr(self, 'name', 'unknown'))
            if self.trains_out_model:
                self.trains_out_model[-1].config_dict = config
            else:
                # todo: support multiple models for the same task
                self.trains_out_model.append(OutputModel(
                    task=PatchKerasModelIO.__main_task,
                    config_dict=config,
                    name=PatchKerasModelIO.__main_task.name + ' ' + model_name_id,
                    label_enumeration=PatchKerasModelIO.__main_task.get_labels_enumeration(),
                    framework=Framework.keras,
                ))
        except Exception as ex:
            LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))

        return config

    @staticmethod
    def _from_config(original_fn, *args, **kwargs):
        try:
            self = original_fn(*args, **kwargs)
        except Exception as ex:
            if not running_remotely():
                raise ex
            self = _Empty()

        # check if we have main task
        if PatchKerasModelIO.__main_task is None:
            return self

        try:
            # check if object already has InputModel
            if not hasattr(self, 'trains_in_model'):
                self.trains_in_model = None

            # get config
            config_dict = kwargs['config'] if 'config' in kwargs else args[0]
            # check if object already has InputModel
            self.trains_in_model = InputModel.empty(
                config_dict=config_dict,
                label_enumeration=PatchKerasModelIO.__main_task.get_labels_enumeration(),
            )
            # todo: support multiple models for the same task
            PatchKerasModelIO.__main_task.connect(self.trains_in_model)
            # if we are running remotely we should deserialize the object
            # because someone might have changed the configuration
            # Hack: disabled
            if False and running_remotely():
                # reload the model
                model_config = self.trains_in_model.config_dict
                # verify that this is the same model so we are not deserializing a diff model
                if (config_dict and config_dict.get('config') and model_config and model_config.get('config') and
                    config_dict.get('config').get('name') == model_config.get('config').get('name')) or \
                        (not config_dict and not model_config):
                    if 'config' in kwargs:
                        kwargs['config'] = model_config
                    else:
                        args = (model_config,) + args[1:]
                    model = original_fn(*args, **kwargs)
                    model.trains_in_model = self.trains_in_model
                    return model

        except Exception as ex:
            LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))

        return self

    @staticmethod
    def _load_weights(original_fn, self, *args, **kwargs):
        # check if we have main task
        if PatchKerasModelIO.__main_task is None:
            return original_fn(self, *args, **kwargs)

        # get filepath
        if self and getattr(self, 'filename', None):
            filepath = getattr(self, 'filename', None)
        else:
            filepath = kwargs['filepath'] if 'filepath' in kwargs else args[0]

        # Hack: disabled
        if False and running_remotely():
            # register/load model weights
            filepath = WeightsFileHandler.restore_weights_file(self, filepath, Framework.keras,
                                                               PatchKerasModelIO.__main_task)
            if 'filepath' in kwargs:
                kwargs['filepath'] = filepath
            else:
                args = (filepath,) + args[1:]
            # load model
            return original_fn(self, *args, **kwargs)

        # try to load the files, if something happened exception will be raised before we register the file
        model = original_fn(self, *args, **kwargs)
        # register/load model weights
        WeightsFileHandler.restore_weights_file(self, filepath, Framework.keras, PatchKerasModelIO.__main_task)
        return model

    @staticmethod
    def _save(original_fn, self, *args, **kwargs):
        if hasattr(self, 'trains_out_model') and self.trains_out_model:
            # noinspection PyProtectedMember
            self.trains_out_model[-1]._processed = False
        original_fn(self, *args, **kwargs)
        # no need to specially call, because the original save uses "save_model" which we overload
        # noinspection PyProtectedMember
        if not hasattr(self, 'trains_out_model') or not self.trains_out_model or \
                not hasattr(self.trains_out_model[-1], '_processed') or not self.trains_out_model[-1]._processed:
            PatchKerasModelIO._update_outputmodel(self, *args, **kwargs)

    @staticmethod
    def _save_weights(original_fn, self, *args, **kwargs):
        original_fn(self, *args, **kwargs)
        PatchKerasModelIO._update_outputmodel(self, *args, **kwargs)

    @staticmethod
    def _update_outputmodel(self, *args, **kwargs):
        # check if we have main task
        if PatchKerasModelIO.__main_task is None:
            return

        try:
            # get filepath
            if self and getattr(self, 'filename', None):
                filepath = getattr(self, 'filename', None)
            else:
                filepath = kwargs['filepath'] if 'filepath' in kwargs else args[0]

            # this will already generate an output model
            # noinspection PyBroadException
            try:
                config = self._updated_config()
            except Exception:
                # we failed to convert the network to json, for some reason (most likely internal keras error)
                config = {}

            if filepath:
                WeightsFileHandler.create_output_model(
                    self, filepath, Framework.keras, PatchKerasModelIO.__main_task,
                    config_obj=config or None, singlefile=True)

        except Exception as ex:
            LoggerRoot.get_base_logger(TensorflowBinding).warning(str(ex))

    @staticmethod
    def _save_model(original_fn, model, filepath, *args, **kwargs):
        original_fn(model, filepath, *args, **kwargs)
        if PatchKerasModelIO.__main_task:
            PatchKerasModelIO._update_outputmodel(model, filepath)

    @staticmethod
    def _load_model(original_fn, filepath, *args, **kwargs):
        if not PatchKerasModelIO.__main_task:
            return original_fn(filepath, *args, **kwargs)

        empty = _Empty()
        # Hack: disabled
        if False and running_remotely():
            # register/load model weights
            filepath = WeightsFileHandler.restore_weights_file(empty, filepath, Framework.keras,
                                                               PatchKerasModelIO.__main_task)
            model = original_fn(filepath, *args, **kwargs)
        else:
            model = original_fn(filepath, *args, **kwargs)
            # register/load model weights
            WeightsFileHandler.restore_weights_file(empty, filepath, Framework.keras, PatchKerasModelIO.__main_task)
        # update the input model object
        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass

        return model


class PatchTensorflowModelIO(object):
    __main_task = None
    __patched = None

    @staticmethod
    def update_current_task(task, **_):
        PatchTensorflowModelIO.__main_task = task
        PatchTensorflowModelIO._patch_model_checkpoint()
        PostImportHookPatching.add_on_import('tensorflow', PatchTensorflowModelIO._patch_model_checkpoint)

    @staticmethod
    def _patch_model_checkpoint():
        if PatchTensorflowModelIO.__patched:
            return

        if 'tensorflow' not in sys.modules:
            return

        PatchTensorflowModelIO.__patched = True
        # noinspection PyBroadException
        try:
            # hack: make sure tensorflow.__init__ is called
            import tensorflow  # noqa
            # noinspection PyUnresolvedReferences
            from tensorflow.python.training.saver import Saver  # noqa
            # noinspection PyBroadException
            try:
                Saver.save = _patched_call(Saver.save, PatchTensorflowModelIO._save)
            except Exception:
                pass
            # noinspection PyBroadException
            try:
                Saver.restore = _patched_call(Saver.restore, PatchTensorflowModelIO._restore)
            except Exception:
                pass
        except ImportError:
            pass
        except Exception:
            LoggerRoot.get_base_logger(TensorflowBinding).debug('Failed patching tensorflow')

        # noinspection PyBroadException
        try:
            # make sure we import the correct version of save
            import tensorflow  # noqa
            from tensorflow.saved_model import save  # noqa
            # actual import
            from tensorflow.python.saved_model import save as saved_model  # noqa
        except ImportError:
            # noinspection PyBroadException
            try:
                # make sure we import the correct version of save
                import tensorflow  # noqa
                from tensorflow.saved_model.experimental import save  # noqa
                # actual import
                import tensorflow.saved_model.experimental as saved_model  # noqa
            except ImportError:
                saved_model = None
            except Exception:
                saved_model = None

        except Exception:
            saved_model = None

        if saved_model is not None:
            saved_model.save = _patched_call(saved_model.save, PatchTensorflowModelIO._save_model)

        # noinspection PyBroadException
        try:
            # make sure we import the correct version of save
            import tensorflow  # noqa
            # actual import
            from tensorflow.saved_model import load  # noqa
            # noinspection PyUnresolvedReferences
            import tensorflow.saved_model as saved_model_load  # noqa
            saved_model_load.load = _patched_call(
                saved_model_load.load,
                PatchTensorflowModelIO._load
                if int(tensorflow.__version__.partition(".")[0]) >= 2
                else PatchTensorflowModelIO._load_lt_2_0
            )
        except ImportError:
            pass
        except Exception:
            LoggerRoot.get_base_logger(TensorflowBinding).debug('Failed patching tensorflow')

        # noinspection PyBroadException
        try:
            # make sure we import the correct version of save
            import tensorflow  # noqa
            # actual import
            # noinspection PyUnresolvedReferences
            from tensorflow.saved_model import loader as loader1  # noqa
            loader1.load = _patched_call(loader1.load, PatchTensorflowModelIO._load)
        except ImportError:
            pass
        except Exception:
            LoggerRoot.get_base_logger(TensorflowBinding).debug('Failed patching tensorflow')

        # noinspection PyBroadException
        try:
            # make sure we import the correct version of save
            import tensorflow  # noqa
            # actual import
            # noinspection PyUnresolvedReferences
            from tensorflow.compat.v1.saved_model import loader as loader2  # noqa
            loader2.load = _patched_call(loader2.load, PatchTensorflowModelIO._load)
        except ImportError:
            pass
        except Exception:
            LoggerRoot.get_base_logger(TensorflowBinding).debug('Failed patching tensorflow')

        # noinspection PyBroadException
        try:
            import tensorflow  # noqa
            from tensorflow.train import Checkpoint  # noqa
            # noinspection PyBroadException
            try:
                Checkpoint.save = _patched_call(Checkpoint.save, PatchTensorflowModelIO._ckpt_save)
            except Exception:
                pass
            # noinspection PyBroadException
            try:
                Checkpoint.restore = _patched_call(Checkpoint.restore, PatchTensorflowModelIO._ckpt_restore)
            except Exception:
                pass
            # noinspection PyBroadException
            try:
                Checkpoint.write = _patched_call(Checkpoint.write, PatchTensorflowModelIO._ckpt_write)
            except Exception:
                pass
        except ImportError:
            pass
        except Exception:
            LoggerRoot.get_base_logger(TensorflowBinding).debug('Failed patching tensorflow')

    @staticmethod
    def _save(original_fn, self, sess, save_path, *args, **kwargs):
        saved_path = original_fn(self, sess, save_path, *args, **kwargs)
        if not saved_path:
            return saved_path
        # store output Model
        return WeightsFileHandler.create_output_model(self, saved_path, Framework.tensorflow,
                                                      PatchTensorflowModelIO.__main_task)

    @staticmethod
    def _save_model(original_fn, obj, export_dir, *args, **kwargs):
        original_fn(obj, export_dir, *args, **kwargs)
        # store output Model
        WeightsFileHandler.create_output_model(obj, export_dir, Framework.tensorflow,
                                               PatchTensorflowModelIO.__main_task)

    @staticmethod
    def _restore(original_fn, self, sess, save_path, *args, **kwargs):
        if PatchTensorflowModelIO.__main_task is None:
            return original_fn(self, sess, save_path, *args, **kwargs)

        # Hack: disabled
        if False and running_remotely():
            # register/load model weights
            save_path = WeightsFileHandler.restore_weights_file(self, save_path, Framework.tensorflow,
                                                                PatchTensorflowModelIO.__main_task)
            # load model
            return original_fn(self, sess, save_path, *args, **kwargs)

        # load model, if something is wrong, exception will be raised before we register the input model
        model = original_fn(self, sess, save_path, *args, **kwargs)
        # register/load model weights
        WeightsFileHandler.restore_weights_file(self, save_path, Framework.tensorflow,
                                                PatchTensorflowModelIO.__main_task)
        return model

    @staticmethod
    def _load_lt_2_0(original_fn, sess, tags=None, export_dir=None, *args, **saver_kwargs):
        if PatchTensorflowModelIO.__main_task is None:
            return original_fn(sess, tags, export_dir, *args, **saver_kwargs)

        # register input model
        empty = _Empty()
        # Hack: disabled
        if False and running_remotely():
            export_dir = WeightsFileHandler.restore_weights_file(
                empty, export_dir, Framework.tensorflow,
                PatchTensorflowModelIO.__main_task
            )
            model = original_fn(sess, tags, export_dir, *args, **saver_kwargs)
        else:
            # try to load model before registering, it might fail
            model = original_fn(sess, tags, export_dir, *args, **saver_kwargs)
            WeightsFileHandler.restore_weights_file(
                empty, export_dir, Framework.tensorflow,
                PatchTensorflowModelIO.__main_task
            )

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass
        return model

    @staticmethod
    def _load(original_fn, export_dir, *args, **saver_kwargs):
        if PatchTensorflowModelIO.__main_task is None:
            return original_fn(export_dir, *args, **saver_kwargs)

        # register input model
        empty = _Empty()
        # Hack: disabled
        if False and running_remotely():
            export_dir = WeightsFileHandler.restore_weights_file(
                empty, export_dir, Framework.tensorflow,
                PatchTensorflowModelIO.__main_task
            )
            model = original_fn(export_dir, *args, **saver_kwargs)
        else:
            # try to load model before registering, it might fail
            model = original_fn(export_dir, *args, **saver_kwargs)
            WeightsFileHandler.restore_weights_file(
                empty, export_dir, Framework.tensorflow,
                PatchTensorflowModelIO.__main_task
            )

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass
        return model

    @staticmethod
    def _ckpt_save(original_fn, self, file_prefix, *args, **kwargs):
        checkpoint_path = original_fn(self, file_prefix, *args, **kwargs)
        if PatchTensorflowModelIO.__main_task is None:
            return checkpoint_path
        WeightsFileHandler.create_output_model(self, checkpoint_path, Framework.tensorflow,
                                               PatchTensorflowModelIO.__main_task)
        return checkpoint_path

    @staticmethod
    def _ckpt_write(original_fn, self, file_prefix, *args, **kwargs):
        checkpoint_path = original_fn(self, file_prefix, *args, **kwargs)
        if PatchTensorflowModelIO.__main_task is None:
            return checkpoint_path
        WeightsFileHandler.create_output_model(self, checkpoint_path, Framework.tensorflow,
                                               PatchTensorflowModelIO.__main_task)
        return checkpoint_path

    @staticmethod
    def _ckpt_restore(original_fn, self, save_path, *args, **kwargs):
        if PatchTensorflowModelIO.__main_task is None:
            return original_fn(self, save_path, *args, **kwargs)

        # register input model
        empty = _Empty()
        # Hack: disabled
        if False and running_remotely():
            save_path = WeightsFileHandler.restore_weights_file(empty, save_path, Framework.tensorflow,
                                                                PatchTensorflowModelIO.__main_task)
            model = original_fn(self, save_path, *args, **kwargs)
        else:
            # try to load model before registering it, in case it fails.
            model = original_fn(self, save_path, *args, **kwargs)
            WeightsFileHandler.restore_weights_file(empty, save_path, Framework.tensorflow,
                                                    PatchTensorflowModelIO.__main_task)

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass
        return model


class PatchTensorflow2ModelIO(object):
    __main_task = None
    __patched = None

    @staticmethod
    def update_current_task(task, **_):
        PatchTensorflow2ModelIO.__main_task = task
        PatchTensorflow2ModelIO._patch_model_checkpoint()
        PostImportHookPatching.add_on_import('tensorflow', PatchTensorflow2ModelIO._patch_model_checkpoint)

    @staticmethod
    def _patch_model_checkpoint():
        if PatchTensorflow2ModelIO.__patched:
            return

        if 'tensorflow' not in sys.modules:
            return

        PatchTensorflow2ModelIO.__patched = True
        # noinspection PyBroadException
        try:
            # hack: make sure tensorflow.__init__ is called
            import tensorflow  # noqa
            from tensorflow.python.training.tracking import util  # noqa
            # noinspection PyBroadException
            try:
                util.TrackableSaver.save = _patched_call(util.TrackableSaver.save,
                                                         PatchTensorflow2ModelIO._save)
            except Exception:
                pass
            # noinspection PyBroadException
            try:
                util.TrackableSaver.restore = _patched_call(util.TrackableSaver.restore,
                                                            PatchTensorflow2ModelIO._restore)
            except Exception:
                pass
        except ImportError:
            pass
        except Exception:
            LoggerRoot.get_base_logger(TensorflowBinding).debug('Failed patching tensorflow v2')

    @staticmethod
    def _save(original_fn, self, file_prefix, *args, **kwargs):
        model = original_fn(self, file_prefix, *args, **kwargs)
        # store output Model
        # noinspection PyBroadException
        try:
            WeightsFileHandler.create_output_model(self, file_prefix, Framework.tensorflow,
                                                   PatchTensorflow2ModelIO.__main_task)
        except Exception:
            pass
        return model

    @staticmethod
    def _restore(original_fn, self, save_path, *args, **kwargs):
        if PatchTensorflow2ModelIO.__main_task is None:
            return original_fn(self, save_path, *args, **kwargs)

        # Hack: disabled
        if False and running_remotely():
            # register/load model weights
            # noinspection PyBroadException
            try:
                save_path = WeightsFileHandler.restore_weights_file(self, save_path, Framework.tensorflow,
                                                                    PatchTensorflow2ModelIO.__main_task)
            except Exception:
                pass
            # load model
            return original_fn(self, save_path, *args, **kwargs)

        # load model, if something is wrong, exception will be raised before we register the input model
        model = original_fn(self, save_path, *args, **kwargs)
        # register/load model weights
        # noinspection PyBroadException
        try:
            WeightsFileHandler.restore_weights_file(self, save_path, Framework.tensorflow,
                                                    PatchTensorflow2ModelIO.__main_task)
        except Exception:
            pass
        return model


# clearml/binding/frameworks/xgboost_bind.py

import sys

import six
from pathlib2 import Path

from ..frameworks.base_bind import PatchBaseModelIO
from ..frameworks import _patched_call, WeightsFileHandler, _Empty
from ..import_bind import PostImportHookPatching
from ...config import running_remotely
from ...model import Framework


class PatchXGBoostModelIO(PatchBaseModelIO):
    __main_task = None
    __patched = None
    __callback_cls = None

    @staticmethod
    def update_current_task(task, **kwargs):
        PatchXGBoostModelIO.__main_task = task
        PatchXGBoostModelIO._patch_model_io()
        PostImportHookPatching.add_on_import('xgboost', PatchXGBoostModelIO._patch_model_io)

    @staticmethod
    def _patch_model_io():
        if PatchXGBoostModelIO.__patched:
            return

        if 'xgboost' not in sys.modules:
            return
        PatchXGBoostModelIO.__patched = True
        # noinspection PyBroadException
        try:
            import xgboost as xgb  # noqa
            bst = xgb.Booster
            bst.save_model = _patched_call(bst.save_model, PatchXGBoostModelIO._save)
            bst.load_model = _patched_call(bst.load_model, PatchXGBoostModelIO._load)
            # noinspection PyBroadException
            try:
                from xgboost.callback import TrainingCallback  # noqa
                PatchXGBoostModelIO.__callback_cls = PatchXGBoostModelIO._generate_training_callback_class()
                xgb.train = _patched_call(xgb.train, PatchXGBoostModelIO._train)
                xgb.training.train = _patched_call(xgb.training.train, PatchXGBoostModelIO._train)
                xgb.sklearn.train = _patched_call(xgb.sklearn.train, PatchXGBoostModelIO._train)
            except ImportError:
                pass
            except Exception:
                pass

        except ImportError:
            pass
        except Exception:
            pass

    @staticmethod
    def _save(original_fn, obj, f, *args, **kwargs):
        ret = original_fn(obj, f, *args, **kwargs)
        if not PatchXGBoostModelIO.__main_task:
            return ret

        if isinstance(f, six.string_types):
            filename = f
        elif hasattr(f, 'name'):
            filename = f.name
            # noinspection PyBroadException
            try:
                f.flush()
            except Exception:
                pass
        else:
            filename = None

        # give the model a descriptive name based on the file name
        # noinspection PyBroadException
        try:
            model_name = Path(filename).stem
        except Exception:
            model_name = None
        WeightsFileHandler.create_output_model(obj, filename, Framework.xgboost, PatchXGBoostModelIO.__main_task,
                                               singlefile=True, model_name=model_name)
        return ret

    @staticmethod
    def _load(original_fn, f, *args, **kwargs):
        if isinstance(f, six.string_types):
            filename = f
        elif hasattr(f, 'name'):
            filename = f.name
        elif len(args) == 1 and isinstance(args[0], six.string_types):
            filename = args[0]
        else:
            filename = None

        if not PatchXGBoostModelIO.__main_task:
            return original_fn(f, *args, **kwargs)

        # register input model
        empty = _Empty()
        # Hack: disabled
        if False and running_remotely():
            filename = WeightsFileHandler.restore_weights_file(empty, filename, Framework.xgboost,
                                                               PatchXGBoostModelIO.__main_task)
            model = original_fn(filename or f, *args, **kwargs)
        else:
            # try to load model before registering, in case we fail
            model = original_fn(f, *args, **kwargs)
            WeightsFileHandler.restore_weights_file(empty, filename, Framework.xgboost,
                                                    PatchXGBoostModelIO.__main_task)

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass
        return model

    @staticmethod
    def _train(original_fn, *args, **kwargs):
        if PatchXGBoostModelIO.__callback_cls:
            callbacks = kwargs.get('callbacks') or []
            kwargs['callbacks'] = callbacks + [
                PatchXGBoostModelIO.__callback_cls(task=PatchXGBoostModelIO.__main_task)
            ]
        return original_fn(*args, **kwargs)

    @classmethod
    def _generate_training_callback_class(cls):
        try:
            from xgboost.callback import TrainingCallback  # noqa
        except ImportError:
            return None

        class ClearMLCallback(TrainingCallback):
            """
            Log evaluation result at each iteration.
            """

            def __init__(self, task, period=1):
                self.period = period
                assert period > 0
                self._last_eval = None
                self._last_eval_epoch = None
                self._logger = task.get_logger()
                super(ClearMLCallback, self).__init__()

            def after_iteration(self, model, epoch, evals_log):
                """ Run after each iteration.  Return True when training should stop. """
                if not evals_log:
                    return False

                if not (self.period == 1 or (epoch % self.period) == 0):
                    self._last_eval = evals_log
                    self._last_eval_epoch = epoch
                    return False

                self._report_eval_log(epoch, evals_log)

                self._last_eval = None
                self._last_eval_epoch = None
                return False

            def after_training(self, model):
                """ Run after training is finished. """
                if self._last_eval:
                    self._report_eval_log(self._last_eval_epoch, self._last_eval)

                return model

            def _report_eval_log(self, epoch, eval_log):
                for data, metric in eval_log.items():
                    for metric_name, log in metric.items():
                        value = log[-1]

                        self._logger.report_scalar(title=data, series=metric_name, value=value, iteration=epoch)

        return ClearMLCallback


# clearml/__init__.py

""" ClearML open SDK """
from six import PY2

from .version import __version__
from .task import Task
from .model import InputModel, OutputModel, Model
from .logger import Logger
from .storage import StorageManager
from .errors import UsageError
from .datasets import Dataset

TaskTypes = Task.TaskTypes

if not PY2:
    from .automation.controller import PipelineController, PipelineDecorator  # noqa: F401

    __all__ = [
        "__version__",
        "Task",
        "TaskTypes",
        "InputModel",
        "OutputModel",
        "Model",
        "Logger",
        "StorageManager",
        "UsageError",
        "Dataset",
        "PipelineController",
        "PipelineDecorator",
    ]
else:
    __all__ = [
        "__version__",
        "Task",
        "TaskTypes",
        "InputModel",
        "OutputModel",
        "Model",
        "Logger",
        "StorageManager",
        "UsageError",
        "Dataset",
    ]


# clearml/errors.py

class UsageError(RuntimeError):
    """ An exception raised for illegal usage of clearml objects"""
    pass


# clearml/logger.py

import logging
import math
import warnings
from typing import Any, Sequence, Union, List, Optional, Tuple, Dict, TYPE_CHECKING

import numpy as np
import six
from PIL import Image
from pathlib2 import Path

from .debugging.log import LoggerRoot

try:
    import pandas as pd
except ImportError:
    pd = None

from .backend_interface.logger import StdStreamPatch
from .backend_interface.task import Task as _Task
from .backend_interface.task.log import TaskHandler
from .backend_interface.util import mutually_exclusive
from .backend_interface.metrics.events import UploadEvent
from .config import running_remotely, get_cache_dir, config, DEBUG_SIMULATE_REMOTE_TASK, deferred_config
from .errors import UsageError
from .storage.helper import StorageHelper
from .utilities.plotly_reporter import SeriesInfo

# Make sure that DeprecationWarning within this package always gets printed
warnings.filterwarnings('always', category=DeprecationWarning, module=__name__)


if TYPE_CHECKING:
    from matplotlib.figure import Figure as MatplotlibFigure  # noqa
    from matplotlib import pyplot  # noqa


class Logger(object):
    """
    The ``Logger`` class is the ClearML console log and metric statistics interface, and contains methods for explicit
    reporting.

    Explicit reporting extends ClearML automagical capturing of inputs and output. Explicit reporting
    methods include scalar plots, line plots, histograms, confusion matrices, 2D and 3D scatter
    diagrams, text logging, tables, and image uploading and reporting.

    In the **ClearML Web-App (UI)**, ``Logger`` output appears in the **RESULTS** tab, **CONSOLE**, **SCALARS**,
    **PLOTS**, and **DEBUG SAMPLES** sub-tabs. When you compare experiments, ``Logger`` output appears in the
    comparisons.

    .. warning::

       Do not construct Logger objects directly.

    You must get a Logger object before calling any of the other ``Logger`` class methods by calling
    :meth:`.Task.get_logger` or :meth:`Logger.current_logger`.


    """
    SeriesInfo = SeriesInfo
    _tensorboard_logging_auto_group_scalars = False
    _tensorboard_single_series_per_graph = deferred_config('metrics.tensorboard_single_series_per_graph', False)

    def __init__(self, private_task, connect_stdout=True, connect_stderr=True, connect_logging=False):
        """
        .. warning::
            **Do not construct Logger manually!**
            Please use :meth:`Logger.get_current`
        """
        assert isinstance(private_task, _Task), \
            'Logger object cannot be instantiated externally, use Logger.current_logger()'
        super(Logger, self).__init__()
        self._task = private_task
        self._default_upload_destination = None
        self._flusher = None
        self._report_worker = None
        self._graph_titles = {}
        self._tensorboard_series_force_prefix = None
        self._task_handler = TaskHandler(task=self._task, capacity=100) \
            if private_task.is_main_task() or (connect_stdout or connect_stderr or connect_logging) else None
        self._connect_std_streams = connect_stdout or connect_stderr
        self._connect_logging = connect_logging

        # Make sure urllib is never in debug/info,
        disable_urllib3_info = config.get('log.disable_urllib3_info', True)
        if disable_urllib3_info and logging.getLogger('urllib3').isEnabledFor(logging.INFO):
            logging.getLogger('urllib3').setLevel(logging.WARNING)

        StdStreamPatch.patch_std_streams(self, connect_stdout=connect_stdout, connect_stderr=connect_stderr)

        if self._connect_logging:
            StdStreamPatch.patch_logging_formatter(self)
        elif not self._connect_std_streams:
            # make sure that at least the main clearml logger is connect
            base_logger = LoggerRoot.get_base_logger()
            if base_logger and base_logger.handlers:
                StdStreamPatch.patch_logging_formatter(self, base_logger.handlers[0])

        self._default_max_sample_history = None

    @classmethod
    def current_logger(cls):
        # type: () -> Logger
        """
        Get the Logger object for the main execution Task, the current running Task, if one exists. If no Logger object
        exists, this method creates one and returns it. Therefore, you can call this method from anywhere
        in the code.

        .. code-block:: py

           logger = Logger.current_logger()

        :return: The Logger object (a singleton) for the current running Task.
        """
        from .task import Task
        task = Task.current_task()
        if not task:
            return None
        return task.get_logger()

    def report_text(self, msg, level=logging.INFO, print_console=True, *args, **_):
        # type: (str, int, bool, Any, Any) -> None
        """
        For explicit reporting, print text to the log. Optionally, print a log level and print to the console.

        For example:

        .. code-block:: py

           logger.report_text('log some text', level=logging.DEBUG, print_console=False)

        You can view the reported text in the **ClearML Web-App (UI)**, **RESULTS** tab, **CONSOLE** sub-tab.

        :param str msg: The text to log.
        :param int level: The log level from the Python ``logging`` package. The default value is ``logging.INFO``.
        :param bool print_console: In addition to the log, print to the console

            The values are:

            - ``True`` - Print to the console. (default)
            - ``False`` - Do not print to the console.
        """
        force_send = not print_console and self._parse_level(level) >= logging.WARNING
        return self._console(msg, level, not print_console, force_send=force_send, *args, **_)

    def report_scalar(self, title, series, value, iteration):
        # type: (str, str, float, int) -> None
        """
        For explicit reporting, plot a scalar series.

        For example, plot a scalar series:

        .. code-block:: py

           logger = Logger.current_logger()
           scalar_series = [random.randint(0,10) for i in range(10)]
           for iteration in range(10):
               logger.report_scalar(
                   title='scalar metrics', series='series', value=scalar_series[iteration], iteration=iteration
               )

        You can view the scalar plots in the **ClearML Web-App (UI)**, **RESULTS** tab, **SCALARS** sub-tab.

        :param str title: The title (metric) of the plot. Plot more than one scalar series on the same plot by using
            the same ``title`` for each call to this method.
        :param str series: The series name (variant) of the reported scalar.
        :param float value: The value to plot per iteration.
        :param int iteration: The reported iteration / step (x-axis of the reported time series)
        """

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_scalar(title=title, series=series, value=float(value), iter=iteration)

    def report_vector(
            self,
            title,  # type: str
            series,  # type: str
            values,  # type: Sequence[Union[int, float]]
            iteration=None,  # type: Optional[int]
            labels=None,  # type: Optional[List[str]]
            xlabels=None,  # type: Optional[List[str]]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            mode=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a vector as (default stacked) histogram.

        For example:

        .. code-block:: py

           vector_series = np.random.randint(10, size=10).reshape(2,5)
           logger.report_vector(title='vector example', series='vector series', values=vector_series, iteration=0,
                labels=['A','B'], xaxis='X axis label', yaxis='Y axis label')

        You can view the vectors plots in the **ClearML Web-App (UI)**, **RESULTS** tab, **PLOTS** sub-tab.

        :param title: The title (metric) of the plot.
        :param series: The series name (variant) of the reported histogram.
        :param values: The series values. A list of floats, or an N-dimensional Numpy array containing
            data for each histogram bar.
        :param iteration: The reported iteration / step. Each ``iteration`` creates another plot.
        :param labels: Labels for each bar group, creating a plot legend labeling each series. (Optional)
        :param xlabels: Labels per entry in each bucket in the histogram (vector), creating a set of labels
            for each histogram bar on the x-axis. (Optional)
        :param xaxis: The x-axis title. (Optional)
        :param yaxis: The y-axis title. (Optional)
        :param mode: Multiple histograms mode, stack / group / relative. Default is 'group'.
        :param extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/bar/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """
        self._touch_title_series(title, series)
        return self.report_histogram(title, series, values, iteration or 0, labels=labels, xlabels=xlabels,
                                     xaxis=xaxis, yaxis=yaxis, mode=mode, extra_layout=extra_layout)

    def report_histogram(
            self,
            title,  # type: str
            series,  # type: str
            values,  # type: Sequence[Union[int, float]]
            iteration=None,  # type: Optional[int]
            labels=None,  # type: Optional[List[str]]
            xlabels=None,  # type: Optional[List[str]]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            mode=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a (default grouped) histogram.
        Notice this function will not calculate the histogram,
        it assumes the histogram was already calculated in `values`

        For example:

        .. code-block:: py

           vector_series = np.random.randint(10, size=10).reshape(2,5)
           logger.report_histogram(title='histogram example', series='histogram series',
                values=vector_series, iteration=0, labels=['A','B'], xaxis='X axis label', yaxis='Y axis label')

        You can view the reported histograms in the **ClearML Web-App (UI)**, **RESULTS** tab, **PLOTS** sub-tab.

        :param title: The title (metric) of the plot.
        :param series: The series name (variant) of the reported histogram.
        :param values: The series values. A list of floats, or an N-dimensional Numpy array containing
            data for each histogram bar.
        :param iteration: The reported iteration / step. Each ``iteration`` creates another plot.
        :param labels: Labels for each bar group, creating a plot legend labeling each series. (Optional)
        :param xlabels: Labels per entry in each bucket in the histogram (vector), creating a set of labels
            for each histogram bar on the x-axis. (Optional)
        :param xaxis: The x-axis title. (Optional)
        :param yaxis: The y-axis title. (Optional)
        :param mode: Multiple histograms mode, stack / group / relative. Default is 'group'.
        :param extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/bar/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        if not isinstance(values, np.ndarray):
            values = np.array(values)

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_histogram(
            title=title,
            series=series,
            histogram=values,
            iter=iteration or 0,
            labels=labels,
            xlabels=xlabels,
            xtitle=xaxis,
            ytitle=yaxis,
            mode=mode or 'group',
            layout_config=extra_layout,
        )

    def report_table(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            table_plot=None,  # type: Optional[pd.DataFrame, Sequence[Sequence]]
            csv=None,  # type: Optional[str]
            url=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit report, report a table plot.

        One and only one of the following parameters must be provided.

        - ``table_plot`` - Pandas DataFrame or Table as list of rows (list)
        - ``csv`` - CSV file
        - ``url`` - URL to CSV file

        For example:

        .. code-block:: py

           df = pd.DataFrame({'num_legs': [2, 4, 8, 0],
                   'num_wings': [2, 0, 0, 0],
                   'num_specimen_seen': [10, 2, 1, 8]},
                   index=['falcon', 'dog', 'spider', 'fish'])

           logger.report_table(title='table example',series='pandas DataFrame',iteration=0,table_plot=df)

        You can view the reported tables in the **ClearML Web-App (UI)**, **RESULTS** tab, **PLOTS** sub-tab.

        :param title: The title (metric) of the table.
        :param series: The series name (variant) of the reported table.
        :param iteration: The reported iteration / step.
        :param table_plot: The output table plot object
        :param csv: path to local csv file
        :param url: A URL to the location of csv file.
        :param extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/table/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """
        mutually_exclusive(
            UsageError, _check_none=True,
            table_plot=table_plot, csv=csv, url=url
        )
        table = table_plot
        if url or csv:
            if not pd:
                raise UsageError(
                    "pandas is required in order to support reporting tables using CSV or a URL, "
                    "please install the pandas python package"
                )
            if url:
                table = pd.read_csv(url, index_col=[0])
            elif csv:
                table = pd.read_csv(csv, index_col=[0])

        def replace(dst, *srcs):
            for src in srcs:
                reporter_table.replace(src, dst, inplace=True)

        if isinstance(table, (list, tuple)):
            reporter_table = table
        else:
            reporter_table = table.fillna(str(np.nan))
            replace("NaN", np.nan, math.nan if six.PY3 else float("nan"))
            replace("Inf", np.inf, math.inf if six.PY3 else float("inf"))
            replace("-Inf", -np.inf, np.NINF, -math.inf if six.PY3 else -float("inf"))
        # noinspection PyProtectedMember
        return self._task._reporter.report_table(
            title=title,
            series=series,
            table=reporter_table,
            iteration=iteration or 0,
            layout_config=extra_layout,
        )

    def report_line_plot(
            self,
            title,  # type: str
            series,  # type: Sequence[SeriesInfo]
            xaxis,  # type: str
            yaxis,  # type: str
            mode='lines',  # type: str
            iteration=None,  # type: Optional[int]
            reverse_xaxis=False,  # type: bool
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot one or more series as lines.

        :param str title: The title (metric) of the plot.
        :param list series: All the series data, one list element for each line in the plot.
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param str mode: The type of line plot.

            The values are:

            - ``lines`` (default)
            - ``markers``
            - ``lines+markers``

        :param bool reverse_xaxis: Reverse the x-axis

            The values are:

            - ``True`` - The x-axis is high to low  (reversed).
            - ``False`` - The x-axis is low to high  (not reversed). (default)

        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/scatter/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        # noinspection PyArgumentList
        series = [self.SeriesInfo(**s) if isinstance(s, dict) else s for s in series]

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series[0].name if series else '')
        # noinspection PyProtectedMember
        return self._task._reporter.report_line_plot(
            title=title,
            series=series,
            iter=iteration or 0,
            xtitle=xaxis,
            ytitle=yaxis,
            mode=mode,
            reverse_xaxis=reverse_xaxis,
            comment=comment,
            layout_config=extra_layout,
        )

    def report_scatter2d(
            self,
            title,  # type: str
            series,  # type: str
            scatter,  # type: Union[Sequence[Tuple[float, float]], np.ndarray]
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            labels=None,  # type: Optional[List[str]]
            mode='lines',  # type: str
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, report a 2d scatter plot.

        For example:

        .. code-block:: py

           scatter2d = np.hstack((np.atleast_2d(np.arange(0, 10)).T, np.random.randint(10, size=(10, 1))))
           logger.report_scatter2d(title="example_scatter", series="series", iteration=0, scatter=scatter2d,
                xaxis="title x", yaxis="title y")

        Plot multiple 2D scatter series on the same plot by passing the same ``title`` and ``iteration`` values
        to this method:

        .. code-block:: py

           scatter2d_1 = np.hstack((np.atleast_2d(np.arange(0, 10)).T, np.random.randint(10, size=(10, 1))))
           logger.report_scatter2d(title="example_scatter", series="series_1", iteration=1, scatter=scatter2d_1,
                xaxis="title x", yaxis="title y")

           scatter2d_2 = np.hstack((np.atleast_2d(np.arange(0, 10)).T, np.random.randint(10, size=(10, 1))))
           logger.report_scatter2d("example_scatter", "series_2", iteration=1, scatter=scatter2d_2,
                xaxis="title x", yaxis="title y")

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported scatter plot.
        :param list scatter: The scatter data. numpy.ndarray or list of (pairs of x,y) scatter:
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param list(str) labels: Labels per point in the data assigned to the ``scatter`` parameter. The labels must be
            in the same order as the data.
        :param str mode: The type of scatter plot.

            The values are:

            - ``lines``
            - ``markers``
            - ``lines+markers``

        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/scatter/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        if not isinstance(scatter, np.ndarray):
            if not isinstance(scatter, list):
                scatter = list(scatter)
            scatter = np.array(scatter)

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_2d_scatter(
            title=title,
            series=series,
            data=scatter,
            iter=iteration or 0,
            mode=mode,
            xtitle=xaxis,
            ytitle=yaxis,
            labels=labels,
            comment=comment,
            layout_config=extra_layout,
        )

    def report_scatter3d(
            self,
            title,  # type: str
            series,  # type: str
            scatter,  # type: Union[Sequence[Tuple[float, float, float]], np.ndarray]
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            zaxis=None,  # type: Optional[str]
            labels=None,  # type: Optional[List[str]]
            mode='markers',  # type: str
            fill=False,  # type: bool
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a 3d scatter graph (with markers).

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported scatter plot.
        :param Union[numpy.ndarray, list] scatter: The scatter data.
            list of (pairs of x,y,z), list of series [[(x1,y1,z1)...]], or numpy.ndarray
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param str zaxis: The z-axis title. (Optional)
        :param list(str) labels: Labels per point in the data assigned to the ``scatter`` parameter. The labels must be
            in the same order as the data.
        :param str mode: The type of scatter plot.

            The values are:

            - ``lines``
            - ``markers``
            - ``lines+markers``

        For example:

        .. code-block:: py

           scatter3d = np.random.randint(10, size=(10, 3))
           logger.report_scatter3d(title="example_scatter_3d", series="series_xyz", iteration=1, scatter=scatter3d,
                xaxis="title x", yaxis="title y", zaxis="title z")

        :param bool fill: Fill the area under the curve

            The values are:

            - ``True`` - Fill
            - ``False`` - Do not fill (default)

        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/scatter3d/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """
        # check if multiple series
        multi_series = (
            isinstance(scatter, list)
            and (
                isinstance(scatter[0], np.ndarray)
                or (
                    scatter[0]
                    and isinstance(scatter[0], list)
                    and isinstance(scatter[0][0], list)
                )
            )
        )

        if not multi_series:
            if not isinstance(scatter, np.ndarray):
                if not isinstance(scatter, list):
                    scatter = list(scatter)
                scatter = np.array(scatter)
            try:
                scatter = scatter.astype(np.float32)
            except ValueError:
                pass

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_3d_scatter(
            title=title,
            series=series,
            data=scatter,
            iter=iteration or 0,
            labels=labels,
            mode=mode,
            fill=fill,
            comment=comment,
            xtitle=xaxis,
            ytitle=yaxis,
            ztitle=zaxis,
            layout_config=extra_layout,
        )

    def report_confusion_matrix(
            self,
            title,  # type: str
            series,  # type: str
            matrix,  # type: np.ndarray
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            xlabels=None,  # type: Optional[List[str]]
            ylabels=None,  # type: Optional[List[str]]
            yaxis_reversed=False,  # type: bool
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a heat-map matrix.

        For example:

        .. code-block:: py

           confusion = np.random.randint(10, size=(10, 10))
           logger.report_confusion_matrix("example confusion matrix", "ignored", iteration=1, matrix=confusion,
                xaxis="title X", yaxis="title Y")

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported confusion matrix.
        :param numpy.ndarray matrix: A heat-map matrix (example: confusion matrix)
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param list(str) xlabels: Labels for each column of the matrix. (Optional)
        :param list(str) ylabels: Labels for each row of the matrix. (Optional)
        :param bool yaxis_reversed: If False 0,0 is at the bottom left corner. If True 0,0 is at the Top left corner
        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/heatmap/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        if not isinstance(matrix, np.ndarray):
            matrix = np.array(matrix)

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_value_matrix(
            title=title,
            series=series,
            data=matrix.astype(np.float32),
            iter=iteration or 0,
            xtitle=xaxis,
            ytitle=yaxis,
            xlabels=xlabels,
            ylabels=ylabels,
            yaxis_reversed=yaxis_reversed,
            comment=comment,
            layout_config=extra_layout,
        )

    def report_matrix(
            self,
            title,  # type: str
            series,  # type: str
            matrix,  # type: np.ndarray
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            xlabels=None,  # type: Optional[List[str]]
            ylabels=None,  # type: Optional[List[str]]
            yaxis_reversed=False,  # type: bool
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a confusion matrix.

        .. note::
            This method is the same as :meth:`Logger.report_confusion_matrix`.

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported confusion matrix.
        :param numpy.ndarray matrix: A heat-map matrix (example: confusion matrix)
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param list(str) xlabels: Labels for each column of the matrix. (Optional)
        :param list(str) ylabels: Labels for each row of the matrix. (Optional)
        :param bool yaxis_reversed: If False 0,0 is at the bottom left corner. If True 0,0 is at the Top left corner
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/heatmap/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """
        self._touch_title_series(title, series)
        return self.report_confusion_matrix(title, series, matrix, iteration or 0,
                                            xaxis=xaxis, yaxis=yaxis, xlabels=xlabels, ylabels=ylabels,
                                            yaxis_reversed=yaxis_reversed,
                                            extra_layout=extra_layout)

    def report_surface(
            self,
            title,  # type: str
            series,  # type: str
            matrix,  # type: np.ndarray
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            zaxis=None,  # type: Optional[str]
            xlabels=None,  # type: Optional[List[str]]
            ylabels=None,  # type: Optional[List[str]]
            camera=None,  # type: Optional[Sequence[float]]
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, report a 3d surface plot.

        .. note::
           This method plots the same data as :meth:`Logger.report_confusion_matrix`, but presents the
           data as a surface diagram not a confusion matrix.

        .. code-block:: py

           surface_matrix = np.random.randint(10, size=(10, 10))
           logger.report_surface("example surface", "series", iteration=0, matrix=surface_matrix,
                xaxis="title X", yaxis="title Y", zaxis="title Z")

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported surface.
        :param numpy.ndarray matrix: A heat-map matrix (example: confusion matrix)
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param str zaxis: The z-axis title. (Optional)
        :param list(str) xlabels: Labels for each column of the matrix. (Optional)
        :param list(str) ylabels: Labels for each row of the matrix. (Optional)
        :param list(float) camera: X,Y,Z coordinates indicating the camera position. The default value is ``(1,1,1)``.
        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/surface/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        if not isinstance(matrix, np.ndarray):
            matrix = np.array(matrix)

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_value_surface(
            title=title,
            series=series,
            data=matrix.astype(np.float32),
            iter=iteration or 0,
            xlabels=xlabels,
            ylabels=ylabels,
            xtitle=xaxis,
            ytitle=yaxis,
            ztitle=zaxis,
            camera=camera,
            comment=comment,
            layout_config=extra_layout,
        )

    def report_image(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            local_path=None,  # type: Optional[str]
            image=None,  # type: Optional[Union[np.ndarray, Image.Image]]
            matrix=None,  # type: Optional[np.ndarray]
            max_image_history=None,  # type: Optional[int]
            delete_after_upload=False,  # type: bool
            url=None  # type: Optional[str]
    ):
        """
        For explicit reporting, report an image and upload its contents.

        This method uploads the image to a preconfigured bucket (see :meth:`Logger.set_default_upload_destination`)
        describing the task ID, title, series and iteration.

        For example:

        .. code-block:: py

           matrix = np.eye(256, 256, dtype=np.uint8)*255
           matrix = np.concatenate((np.atleast_3d(matrix), np.zeros((256, 256, 2), dtype=np.uint8)), axis=2)
           logger.report_image("test case", "image color red", iteration=1, image=m)

           image_open = Image.open(os.path.join("<image_path>", "<image_filename>"))
           logger.report_image("test case", "image PIL", iteration=1, image=image_open)

        One and only one of the following parameters must be provided.

        - ``local_path``
        - ``url``
        - ``image``
        - ``matrix``

        :param title: The title (metric) of the image.
        :param series: The series name (variant) of the reported image.
        :param iteration: The reported iteration / step.
        :param local_path: A path to an image file.
        :param url: A URL for the location of a pre-uploaded image.
        :param image: Image data (RGB).
        :param matrix: Deperacted, Image data (RGB).

            .. note::
               The ``matrix`` parameter is deprecated. Use the ``image`` parameters.
        :param max_image_history: The maximum number of images to store per metric/variant combination.
            For an unlimited number, use a negative value. The default value is set in global configuration
            (default=``5``).
        :param delete_after_upload: After the upload, delete the local copy of the image

            The values are:

            - ``True`` - Delete after upload.
            - ``False`` - Do not delete after upload. (default)
        """
        mutually_exclusive(
            UsageError, _check_none=True,
            local_path=local_path or None, url=url or None, image=image, matrix=matrix
        )
        if matrix is not None:
            warnings.warn("'matrix' variable is deprecated; use 'image' instead.", DeprecationWarning)
        if image is None:
            image = matrix
        if image is not None and not isinstance(image, (np.ndarray, Image.Image)):
            raise ValueError("Supported 'image' types are: numpy.ndarray or PIL.Image")

        # if task was not started, we have to start it
        self._start_task_if_needed()

        self._touch_title_series(title, series)

        if url:
            # noinspection PyProtectedMember
            self._task._reporter.report_image(
                title=title,
                series=series,
                src=url,
                iter=iteration or 0,
            )

        else:
            upload_uri = self.get_default_upload_destination()
            if not upload_uri:
                upload_uri = Path(get_cache_dir()) / 'debug_images'
                upload_uri.mkdir(parents=True, exist_ok=True)
                # Verify that we can upload to this destination
                upload_uri = str(upload_uri)
                storage = StorageHelper.get(upload_uri)
                upload_uri = storage.verify_upload(folder_uri=upload_uri)

            if isinstance(image, Image.Image):
                image = np.array(image)  # noqa
            # noinspection PyProtectedMember
            self._task._reporter.report_image_and_upload(
                title=title,
                series=series,
                path=local_path,
                image=image,
                iter=iteration or 0,
                upload_uri=upload_uri,
                max_image_history=max_image_history if max_image_history is not None
                else self._default_max_sample_history,
                delete_after_upload=delete_after_upload,
            )

    def report_media(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            local_path=None,  # type: Optional[str]
            stream=None,  # type: Optional[Union[six.BytesIO, six.StringIO]]
            file_extension=None,  # type: Optional[str]
            max_history=None,  # type: Optional[int]
            delete_after_upload=False,  # type: bool
            url=None  # type: Optional[str]
    ):
        """
        Report media upload its contents, including images, audio, and video.

        Media is uploaded to a preconfigured bucket (see setup_upload()) with a key (filename)
        describing the task ID, title, series and iteration.

        One and only one of the following parameters must be provided

        - ``local_path``
        - ``stream``
        - ``url``

        If you use ``stream`` for a BytesIO stream to upload, ``file_extension`` must be provided.

        :param str title: The title (metric) of the media.
        :param str series: The series name (variant) of the reported media.
        :param int iteration: The reported iteration / step.
        :param str local_path: A path to an media file.
        :param stream: BytesIO stream to upload. If provided, ``file_extension`` must also be provided.
        :param str url: A URL to the location of a pre-uploaded media.
        :param file_extension: A file extension to use when ``stream`` is passed.
        :param int max_history: The maximum number of media files to store per metric/variant combination
            use negative value for unlimited. default is set in global configuration (default=5)
        :param bool delete_after_upload: After the file is uploaded, delete the local copy

            - ``True`` - Delete
            - ``False`` - Do not delete

        """
        mutually_exclusive(
            UsageError, _check_none=True,
            local_path=local_path or None, url=url or None, stream=stream,
        )
        if stream is not None and not file_extension:
            raise ValueError("No file extension provided for stream media upload")

        # if task was not started, we have to start it
        self._start_task_if_needed()

        self._touch_title_series(title, series)

        if url:
            # noinspection PyProtectedMember
            self._task._reporter.report_media(
                title=title,
                series=series,
                src=url,
                iter=iteration or 0,
            )

        else:
            upload_uri = self.get_default_upload_destination()
            if not upload_uri:
                upload_uri = Path(get_cache_dir()) / 'debug_images'
                upload_uri.mkdir(parents=True, exist_ok=True)
                # Verify that we can upload to this destination
                upload_uri = str(upload_uri)
                storage = StorageHelper.get(upload_uri)
                upload_uri = storage.verify_upload(folder_uri=upload_uri)
            # noinspection PyProtectedMember
            self._task._reporter.report_media_and_upload(
                title=title,
                series=series,
                path=local_path,
                stream=stream,
                iter=iteration or 0,
                upload_uri=upload_uri,
                max_history=max_history if max_history is not None else self._default_max_sample_history,
                delete_after_upload=delete_after_upload,
                file_extension=file_extension,
            )

    def report_plotly(
            self,
            title,  # type: str
            series,  # type: str
            figure,  # type: Union[Dict, "Figure"]  # noqa: F821
            iteration=None,  # type: Optional[int]
    ):
        """
        Report a ``Plotly`` figure (plot) directly

        ``Plotly`` figure can be a ``plotly.graph_objs._figure.Figure`` or a dictionary as defined by ``plotly.js``

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported plot.
        :param int iteration: The reported iteration / step.
        :param dict figure: A ``plotly`` Figure object or a ``poltly`` dictionary
        """
        # if task was not started, we have to start it
        self._start_task_if_needed()

        self._touch_title_series(title, series)

        plot = figure if isinstance(figure, dict) else figure.to_plotly_json()
        # noinspection PyBroadException
        try:
            plot['layout']['title'] = series
        except Exception:
            pass
        # noinspection PyProtectedMember
        self._task._reporter.report_plot(
            title=title,
            series=series,
            plot=plot,
            iter=iteration or 0,
        )

    def report_matplotlib_figure(
            self,
            title,  # type: str
            series,  # type: str
            figure,  # type: Union[MatplotlibFigure, pyplot]
            iteration=None,  # type: Optional[int]
            report_image=False,  # type: bool
            report_interactive=True,  # type: bool
    ):
        """
        Report a ``matplotlib`` figure / plot directly

        ``matplotlib.figure.Figure`` / ``matplotlib.pyplot``

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported plot.
        :param int iteration: The reported iteration / step.
        :param MatplotlibFigure figure: A ``matplotlib`` Figure object
        :param report_image: Default False. If True the plot will be uploaded as a debug sample (png image),
            and will appear under the debug samples tab (instead of the Plots tab).
        :param report_interactive: If True (default) it will try to convert the matplotlib into interactive
            plot in the UI. If False the matplotlib is saved as is and will
            be non-interactive (with the exception of zooming in/out)
        """
        # if task was not started, we have to start it
        self._start_task_if_needed()

        # noinspection PyProtectedMember
        self._task._reporter.report_matplotlib(
            title=title,
            series=series,
            figure=figure,
            iter=iteration or 0,
            logger=self,
            force_save_as_image=False if report_interactive and not report_image
            else ('png' if report_image else True),
        )

    def set_default_upload_destination(self, uri):
        # type: (str) -> None
        """
        Set the destination storage URI (for example, S3, Google Cloud Storage, a file path) for uploading debug images.

        The images are uploaded separately. A link to each image is reported.

        .. note::
           Credentials for the destination storage are specified in the  ClearML configuration file,
           ``~/clearml.conf``.

        :param str uri: example: 's3://bucket/directory/' or 'file:///tmp/debug/'

        :return: True, if the destination scheme is supported (for example, ``s3://``, ``file://``, or ``gc://``).
            False, if not supported.

        """

        # Create the storage helper
        storage = StorageHelper.get(uri)

        # Verify that we can upload to this destination
        uri = storage.verify_upload(folder_uri=uri)

        self._default_upload_destination = uri

    def get_default_upload_destination(self):
        # type: () -> str
        """
        Get the destination storage URI (for example, S3, Google Cloud Storage, a file path) for uploading debug images
        (see :meth:`Logger.set_default_upload_destination`).

        :return: The default upload destination URI.

            For example: ``s3://bucket/directory/``, or ``file:///tmp/debug/``.
        """
        # noinspection PyProtectedMember
        return self._default_upload_destination or self._task._get_default_report_storage_uri()

    def flush(self, wait=False):
        # type: (bool) -> bool
        """
        Flush cached reports and console outputs to backend.

        :param wait: Wait for all outstanding uploads and events to be sent (default False)

        :return: True, if successfully flushed the cache. False, if failed.
        """
        self._flush_stdout_handler()
        if self._task:
            return self._task.flush(wait_for_uploads=wait)
        return False

    def get_flush_period(self):
        # type: () -> Optional[float]
        """
        Get the Logger flush period.

        :return: The logger flush period in seconds.
        """
        if self._flusher:
            return self._flusher.period
        return None

    def set_flush_period(self, period):
        # type: (float) -> None
        """
        Set the logger flush period.

        Deprecated - Use ``sdk.development.worker.report_period_sec`` to externally control the flush period.

        :param float period: The period to flush the logger in seconds. To set no periodic flush,
            specify ``None`` or ``0``.
        """
        pass

    def set_default_debug_sample_history(self, max_history):
        # type: (int) -> None
        """
        Set the default maximum debug sample history when reporting media/debug samples.
        Overrides the configuration file defaults.
        When reporting debug samples with the same title/series combination and running iterations,
        only the last X samples are stored (in other words samples are overwritten).
        The default history size set with `max_history` is used when calling
        `report_image`, `report_media` etc. without specifying `max_history`

        :param max_history: Number of samples (files) to store on a unique set of title/series being reported
            with different iteration counter. This is used to make sure users do not end up exploding storage
            on server storage side.

            For example the following code sample will store the last 5 images even though
            we are reporting 100 samples.

            .. code-block:: py

                logger.set_default_debug_sample_history(5)
                for i in range(100):
                    logger.report_image(title='image', series='sample', iteration=i, ...)

        :return:
        """
        self._default_max_sample_history = int(max_history)

    def get_default_debug_sample_history(self):
        # type: () -> int
        """
        Return the the default max debug sample history when reporting media/debug samples.
        If value was not set specifically, the functions returns the configuration file default value.

        :return: default number of samples (files) to store on a unique set of title/series being reported
            with different iteration counter. This is used to make sure users do not end up exploding storage
            on server storage side.
        """
        if self._default_max_sample_history is not None:
            return self._default_max_sample_history
        # noinspection PyProtectedMember
        return int(UploadEvent._file_history_size)

    def report_image_and_upload(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            path=None,  # type: Optional[str]
            matrix=None,  # type: Optional[Union[np.ndarray, Image.Image]]
            max_image_history=None,  # type: Optional[int]
            delete_after_upload=False  # type: bool
    ):
        """
        .. deprecated:: 0.13.0
            Use :meth:`Logger.report_image` instead
        """
        self.report_image(title=title, series=series, iteration=iteration or 0, local_path=path, image=matrix,
                          max_image_history=max_image_history, delete_after_upload=delete_after_upload)

    def capture_logging(self):
        # type: () -> "_LoggingContext"
        """
        Return context capturing all the logs (via logging) reported under the context

        :return: a ContextManager
        """
        class _LoggingContext(object):
            def __init__(self, a_logger):
                self.logger = a_logger

            def __enter__(self, *_, **__):
                if not self.logger:
                    return
                StdStreamPatch.patch_logging_formatter(self.logger)

            def __exit__(self, *_, **__):
                if not self.logger:
                    return
                StdStreamPatch.remove_patch_logging_formatter()

        # Do nothing if we already have full logging support
        return _LoggingContext(None if self._connect_logging else self)

    @classmethod
    def tensorboard_auto_group_scalars(cls, group_scalars=False):
        # type: (bool) -> None
        """
        Group together TensorBoard scalars that do not have a title, or assign a title/series with the same tag.

        :param group_scalars: Group TensorBoard scalars without a title

            The values are:

            - ``True`` - Scalars without specific titles are grouped together in the "Scalars" plot, preserving
              backward compatibility with ClearML automagical behavior.
            - ``False`` - TensorBoard scalars without titles get a title/series with the same tag. (default)
        """
        cls._tensorboard_logging_auto_group_scalars = group_scalars

    @classmethod
    def tensorboard_single_series_per_graph(cls, single_series=False):
        # type: (bool) -> None
        """
        Deprecated, this is now controlled from the UI!
        Group TensorBoard scalar series together or in separate plots.

        :param single_series: Group TensorBoard scalar series together

            The values are:

            - ``True`` - Generate a separate plot for each TensorBoard scalar series.
            - ``False`` - Group the TensorBoard scalar series together in the same plot. (default)

        """
        cls._tensorboard_single_series_per_graph = single_series

    @classmethod
    def matplotlib_force_report_non_interactive(cls, force):
        # type: (bool) -> None
        """
        If True all matplotlib are always converted to non interactive static plots (images), appearing in under
        the Plots section. If False (default), matplotlib figures are converted into interactive web UI plotly
        figures, in case figure conversion fails, it defaults to non-interactive plots.

        :param force: If True all matplotlib figures are converted automatically to non-interactive plots.
        """
        from clearml.backend_interface.metrics import Reporter
        Reporter.matplotlib_force_report_non_interactive(force=force)

    @classmethod
    def _remove_std_logger(cls):
        # noinspection PyBroadException
        try:
            StdStreamPatch.remove_std_logger()
        except Exception:
            return False
        return True

    def _parse_level(self, level):
        # type: (Any) -> int
        try:
            return int(level)
        except (TypeError, ValueError):
            self._task.log.log(level=logging.ERROR,
                               msg='Logger failed casting log level "%s" to integer' % str(level))
            return logging.INFO

    def _console(self, msg, level=logging.INFO, omit_console=False, force_send=False, *args, **_):
        # type: (str, int, bool, bool, Any, Any) -> None
        """
        print text to log (same as print to console, and also prints to console)

        :param msg: text to print to the console (always send to the backend and displayed in console)
        :param level: logging level, default: logging.INFO
        :param omit_console: Omit the console output, and only send the ``msg`` value to the log
        :param force_send: Report with an explicit log level. Only supported if ``omit_console`` is True

            - ``True`` - Omit the console output.
            - ``False`` - Print the console output. (default)

        """
        level = self._parse_level(level)

        force_send = force_send and omit_console

        # noinspection PyProtectedMember
        if not self._skip_console_log() or not self._task._is_remote_main_task() or force_send:
            # check if we have a TaskHandler and that it is valid (shutdown will clear the .task_id)
            if self._task_handler and self._task_handler.task_id:
                # noinspection PyBroadException
                try:
                    record = self._task.log.makeRecord(
                        "console", level=level, fn='', lno=0, func='', msg=msg, args=args, exc_info=None
                    )
                    # find the task handler that matches our task
                    self._task_handler.emit(record)
                except Exception:
                    # avoid infinite loop, output directly to stderr
                    # noinspection PyBroadException
                    try:
                        # make sure we are writing to the original stdout
                        StdStreamPatch.stderr_original_write(
                            'clearml.Logger failed sending log [level {}]: "{}"\n'.format(level, msg))
                    except Exception:
                        pass
            else:
                # noinspection PyProtectedMember
                self._task._reporter.report_console(message=msg, level=level)

        if not omit_console:
            # if we are here and we grabbed the stdout, we need to print the real thing
            if self._connect_std_streams and not self._skip_console_log():
                # noinspection PyBroadException
                try:
                    # make sure we are writing to the original stdout
                    StdStreamPatch.stdout_original_write(str(msg) + '\n')
                except Exception:
                    pass
            else:
                print(str(msg))

        # if task was not started, we have to start it
        self._start_task_if_needed()

    def _report_image_plot_and_upload(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            path=None,  # type: Optional[str]
            matrix=None,  # type: Optional[np.ndarray]
            max_image_history=None,  # type: Optional[int]
            delete_after_upload=False  # type: bool
    ):
        """
        Report an image, upload its contents, and present in plots section using plotly

        Image is uploaded to a preconfigured bucket (see :meth:`Logger.set_default_upload_destination`)
        describing the task ID, title, series and iteration.

        :param title: Title (AKA metric)
        :param series: Series (AKA variant)
        :param iteration: Iteration number
        :param path: A path to an image file. Required unless matrix is provided.
        :param matrix: A 3D numpy.ndarray object containing image data (RGB). Required unless filename is provided.
        :param max_image_history: maximum number of image to store per metric/variant combination \
        use negative value for unlimited. default is set in global configuration (default=5)
        :param delete_after_upload: if True, one the file was uploaded the local copy will be deleted
        """

        # if task was not started, we have to start it
        self._start_task_if_needed()
        upload_uri = self.get_default_upload_destination()
        if not upload_uri:
            upload_uri = Path(get_cache_dir()) / 'debug_images'
            upload_uri.mkdir(parents=True, exist_ok=True)
            # Verify that we can upload to this destination
            upload_uri = str(upload_uri)
            storage = StorageHelper.get(upload_uri)
            upload_uri = storage.verify_upload(folder_uri=upload_uri)

        # noinspection PyProtectedMember
        self._task._reporter.report_image_plot_and_upload(
            title=title,
            series=series,
            path=path,
            matrix=matrix,
            iter=iteration or 0,
            upload_uri=upload_uri,
            max_image_history=max_image_history if max_image_history is not None else self._default_max_sample_history,
            delete_after_upload=delete_after_upload,
        )

    def _report_file_and_upload(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            path=None,  # type: Optional[str]
            max_file_history=None,  # type: Optional[int]
            delete_after_upload=False  # type: bool
    ):
        """
        Upload a file and report it as link in the debug images section.

        File is uploaded to a preconfigured storage (see :meth:`Loggerset_default_upload_destination`)
        describing the task ID, title, series and iteration.

        :param title: Title (AKA metric)
        :param series: Series (AKA variant)
        :param iteration: Iteration number
        :param path: A path to file to be uploaded
        :param max_file_history: maximum number of files to store per metric/variant combination \
        use negative value for unlimited. default is set in global configuration (default=5)
        :param delete_after_upload: if True, one the file was uploaded the local copy will be deleted
        """

        # if task was not started, we have to start it
        self._start_task_if_needed()
        upload_uri = self.get_default_upload_destination()
        if not upload_uri:
            upload_uri = Path(get_cache_dir()) / 'debug_images'
            upload_uri.mkdir(parents=True, exist_ok=True)
            # Verify that we can upload to this destination
            upload_uri = str(upload_uri)
            storage = StorageHelper.get(upload_uri)
            upload_uri = storage.verify_upload(folder_uri=upload_uri)
        # noinspection PyProtectedMember
        self._task._reporter.report_image_and_upload(
            title=title,
            series=series,
            path=path,
            image=None,
            iter=iteration or 0,
            upload_uri=upload_uri,
            max_image_history=max_file_history if max_file_history is not None else self._default_max_sample_history,
            delete_after_upload=delete_after_upload,
        )

    def _start_task_if_needed(self):
        # deprecated
        pass

    def _flush_stdout_handler(self):
        if self._task_handler:
            self._task_handler.flush()

    def _close_stdout_handler(self, wait=True):
        # detach the sys stdout/stderr
        if self._connect_std_streams:
            StdStreamPatch.remove_std_logger(self)

        if self._task_handler:
            t = self._task_handler
            self._task_handler = None
            t.close(wait)

    def _touch_title_series(self, title, series):
        # type: (str, str) -> None
        if title not in self._graph_titles:
            self._graph_titles[title] = set()
        self._graph_titles[title].add(series)

    def _get_used_title_series(self):
        # type: () -> dict
        return self._graph_titles

    def _get_tensorboard_series_prefix(self):
        # type: () -> Optional[str]
        """
        :return str: return a string prefix to put in front of every report combing from tensorboard
        """
        return self._tensorboard_series_force_prefix

    def _set_tensorboard_series_prefix(self, prefix):
        # type: (Optional[str]) -> ()
        """
        :param str prefix: Set a string prefix to put in front of every report combing from tensorboard
        """
        self._tensorboard_series_force_prefix = str(prefix) if prefix else None

    @classmethod
    def _get_tensorboard_auto_group_scalars(cls):
        # type: () -> bool
        """
        :return: True, if we preserve Tensorboard backward compatibility behaviour,
            i.e., scalars without specific title will be under the "Scalars" graph
            default is False: Tensorboard scalars without title will have title/series with the same tag
        """
        return cls._tensorboard_logging_auto_group_scalars

    @classmethod
    def _get_tensorboard_single_series_per_graph(cls):
        # type: () -> bool
        """
        :return: True, if we generate a separate graph (plot) for each Tensorboard scalar series
            default is False: Tensorboard scalar series will be grouped according to their title
        """
        return cls._tensorboard_single_series_per_graph

    @classmethod
    def _skip_console_log(cls):
        return bool(running_remotely() and not DEBUG_SIMULATE_REMOTE_TASK.get())


# clearml/model.py

import abc
import os
import tarfile
import zipfile
from tempfile import mkdtemp, mkstemp

import six
from typing import List, Dict, Union, Optional, Mapping, TYPE_CHECKING, Sequence

from .backend_api import Session
from .backend_api.services import models, projects
from pathlib2 import Path

from .utilities.config import config_dict_to_text, text_to_config_dict

from .backend_interface.util import validate_dict, get_single_result, mutually_exclusive, exact_match_regex
from .debugging.log import get_logger
from .storage.cache import CacheManager
from .storage.helper import StorageHelper
from .storage.util import get_common_path
from .utilities.enum import Options
from .backend_interface import Task as _Task
from .backend_interface.model import create_dummy_model, Model as _Model
from .backend_interface.session import SendError
from .config import running_remotely, get_cache_dir


if TYPE_CHECKING:
    from .task import Task


class Framework(Options):
    """
    Optional frameworks for output model
    """
    tensorflow = 'TensorFlow'
    tensorflowjs = 'TensorFlow_js'
    tensorflowlite = 'TensorFlow_Lite'
    pytorch = 'PyTorch'
    caffe = 'Caffe'
    caffe2 = 'Caffe2'
    onnx = 'ONNX'
    keras = 'Keras'
    mknet = 'MXNet'
    cntk = 'CNTK'
    torch = 'Torch'
    darknet = 'Darknet'
    paddlepaddle = 'PaddlePaddle'
    scikitlearn = 'ScikitLearn'
    xgboost = 'XGBoost'
    lightgbm = 'LightGBM'
    parquet = 'Parquet'
    megengine = 'MegEngine'
    catboost = 'CatBoost'

    __file_extensions_mapping = {
        '.pb': (tensorflow, tensorflowjs, onnx, ),
        '.meta': (tensorflow, ),
        '.pbtxt': (tensorflow, onnx, ),
        '.zip': (tensorflow, ),
        '.tgz': (tensorflow, ),
        '.tar.gz': (tensorflow, ),
        'model.json': (tensorflowjs, ),
        '.tflite': (tensorflowlite, ),
        '.pth': (pytorch, ),
        '.pt': (pytorch, ),
        '.caffemodel': (caffe, ),
        '.prototxt': (caffe, ),
        'predict_net.pb': (caffe2, ),
        'predict_net.pbtxt': (caffe2, ),
        '.onnx': (onnx, ),
        '.h5': (keras, ),
        '.hdf5': (keras, ),
        '.keras': (keras, ),
        '.model': (mknet, cntk, xgboost),
        '-symbol.json': (mknet, ),
        '.cntk': (cntk, ),
        '.t7': (torch, ),
        '.cfg': (darknet, ),
        '__model__': (paddlepaddle, ),
        '.pkl': (scikitlearn, keras, xgboost, megengine),
        '.parquet': (parquet, ),
        '.cbm': (catboost, ),
    }

    @classmethod
    def _get_file_ext(cls, framework, filename):
        mapping = cls.__file_extensions_mapping
        filename = filename.lower()

        def find_framework_by_ext(framework_selector):
            for ext, frameworks in mapping.items():
                if frameworks and filename.endswith(ext):
                    fw = framework_selector(frameworks)
                    if fw:
                        return fw, ext

        # If no framework, try finding first framework matching the extension, otherwise (or if no match) try matching
        # the given extension to the given framework. If no match return an empty extension
        return (
            (not framework and find_framework_by_ext(lambda frameworks_: frameworks_[0]))
            or find_framework_by_ext(lambda frameworks_: framework if framework in frameworks_ else None)
            or (framework, filename.split('.')[-1] if '.' in filename else '')
        )


@six.add_metaclass(abc.ABCMeta)
class BaseModel(object):
    # noinspection PyProtectedMember
    _archived_tag = _Task.archived_tag
    _package_tag = "package"

    @property
    def id(self):
        # type: () -> str
        """
        The Id (system UUID) of the model.

        :return: The model ID.
        """
        return self._get_model_data().id

    @property
    def name(self):
        # type: () -> str
        """
        The name of the model.

        :return: The model name.
        """
        return self._get_model_data().name

    @name.setter
    def name(self, value):
        # type: (str) -> None
        """
        Set the model name.

        :param str value: The model name.
        """
        self._get_base_model().update(name=value)

    @property
    def comment(self):
        # type: () -> str
        """
        The comment for the model. Also, use for a model description.

        :return: The model comment / description.
        """
        return self._get_model_data().comment

    @comment.setter
    def comment(self, value):
        # type: (str) -> None
        """
        Set comment for the model. Also, use for a model description.

        :param str value: The model comment/description.
        """
        self._get_base_model().update(comment=value)

    @property
    def tags(self):
        # type: () -> List[str]
        """
        A list of tags describing the model.

        :return: The list of tags.
        """
        return self._get_model_data().tags

    @tags.setter
    def tags(self, value):
        # type: (List[str]) -> None
        """
        Set the list of tags describing the model.

        :param value: The tags.

        :type value: list(str)
        """
        self._get_base_model().update(tags=value)

    @property
    def system_tags(self):
        # type: () -> List[str]
        """
        A list of system tags describing the model.

        :return: The list of tags.
        """
        data = self._get_model_data()
        return data.system_tags if Session.check_min_api_version('2.3') else data.tags

    @system_tags.setter
    def system_tags(self, value):
        # type: (List[str]) -> None
        """
        Set the list of system tags describing the model.

        :param value: The tags.

        :type value: list(str)
        """
        self._get_base_model().update(system_tags=value)

    @property
    def config_text(self):
        # type: () -> str
        """
        The configuration as a string. For example, prototxt, an ini file, or Python code to evaluate.

        :return: The configuration.
        """
        # noinspection PyProtectedMember
        return _Model._unwrap_design(self._get_model_data().design)

    @property
    def config_dict(self):
        # type: () -> dict
        """
        The configuration as a dictionary, parsed from the design text. This usually represents the model configuration.
        For example, prototxt, an ini file, or Python code to evaluate.

        :return: The configuration.
        """
        return self._text_to_config_dict(self.config_text)

    @property
    def labels(self):
        # type: () -> Dict[str, int]
        """
        The label enumeration of string (label) to integer (value) pairs.


        :return: A dictionary containing labels enumeration, where the keys are labels and the values as integers.
        """
        return self._get_model_data().labels

    @property
    def task(self):
        # type: () -> str
        """
        Return the creating task ID

        :return: The Task ID (str)
        """
        return self._task.id if self._task else self._get_base_model().task

    @property
    def url(self):
        # type: () -> str
        """
        Return the url of the model file (or archived files)

        :return: The model file URL.
        """
        return self._get_base_model().uri

    @property
    def published(self):
        # type: () -> bool
        return self._get_base_model().locked

    @property
    def framework(self):
        # type: () -> str
        return self._get_model_data().framework

    def __init__(self, task=None):
        # type: (Task) -> None
        super(BaseModel, self).__init__()
        self._log = get_logger()
        self._task = None
        self._set_task(task)

    def get_weights(self, raise_on_error=False):
        # type: (bool) -> str
        """
        Download the base model and return the locally stored filename.

        :param bool raise_on_error: If True and the artifact could not be downloaded,
            raise ValueError, otherwise return None on failure and output log warning.

        :return: The locally stored file.
        """
        # download model (synchronously) and return local file
        return self._get_base_model().download_model_weights(raise_on_error=raise_on_error)

    def get_weights_package(self, return_path=False, raise_on_error=False):
        # type: (bool, bool) -> Optional[Union[str, List[Path]]]
        """
        Download the base model package into a temporary directory (extract the files), or return a list of the
        locally stored filenames.

        :param bool return_path: Return the model weights or a list of filenames (Optional)

            - ``True`` - Download the model weights into a temporary directory, and return the temporary directory path.
            - ``False`` - Return a list of the locally stored filenames. (Default)

        :param bool raise_on_error: If True and the artifact could not be downloaded,
            raise ValueError, otherwise return None on failure and output log warning.

        :return: The model weights, or a list of the locally stored filenames.
            if raise_on_error=False, returns None on error.
        """
        # check if model was packaged
        if not self._is_package():
            raise ValueError('Model is not packaged')

        # download packaged model
        packed_file = self.get_weights(raise_on_error=raise_on_error)

        if not packed_file:
            if raise_on_error:
                raise ValueError('Model package \'{}\' could not be downloaded'.format(self.url))
            return None

        # unpack
        target_folder = mkdtemp(prefix='model_package_')
        if not target_folder:
            raise ValueError('cannot create temporary directory for packed weight files')

        for func in (zipfile.ZipFile, tarfile.open):
            try:
                obj = func(packed_file)
                obj.extractall(path=target_folder)
                break
            except (zipfile.BadZipfile, tarfile.ReadError):
                pass
        else:
            raise ValueError('cannot extract files from packaged model at %s', packed_file)

        if return_path:
            return target_folder

        target_files = list(Path(target_folder).glob('*'))
        return target_files

    def publish(self):
        # type: () -> ()
        """
        Set the model to the status ``published`` and for public use. If the model's status is already ``published``,
        then this method is a no-op.
        """

        if not self.published:
            self._get_base_model().publish()

    def _running_remotely(self):
        # type: () -> ()
        return bool(running_remotely() and self._task is not None)

    def _set_task(self, value):
        # type: (_Task) -> ()
        if value is not None and not isinstance(value, _Task):
            raise ValueError('task argument must be of Task type')
        self._task = value

    @abc.abstractmethod
    def _get_model_data(self):
        pass

    @abc.abstractmethod
    def _get_base_model(self):
        pass

    def _set_package_tag(self):
        if self._package_tag not in self.system_tags:
            self.system_tags.append(self._package_tag)
            self._get_base_model().edit(system_tags=self.system_tags)

    def _is_package(self):
        return self._package_tag in (self.system_tags or [])

    @staticmethod
    def _config_dict_to_text(config):
        if not isinstance(config, six.string_types) and not isinstance(config, dict):
            raise ValueError("Model configuration only supports dictionary or string objects")
        return config_dict_to_text(config)

    @staticmethod
    def _text_to_config_dict(text):
        if not isinstance(text, six.string_types):
            raise ValueError("Model configuration parsing only supports string")
        return text_to_config_dict(text)

    @staticmethod
    def _resolve_config(config_text=None, config_dict=None):
        mutually_exclusive(config_text=config_text, config_dict=config_dict, _require_at_least_one=False)
        if config_dict:
            return InputModel._config_dict_to_text(config_dict)

        return config_text


class Model(BaseModel):
    """
    Represent an existing model in the system, search by model id.
    The Model will be read-only and can be used to pre initialize a network
    """

    def __init__(self, model_id):
        # type: (str) ->None
        """
        Load model based on id, returned object is read-only and can be connected to a task

        Notice, we can override the input model when running remotely

        :param model_id: id (string)
        """
        super(Model, self).__init__()
        self._base_model_id = model_id
        self._base_model = None

    def get_local_copy(self, extract_archive=True, raise_on_error=False):
        # type: (bool, bool) -> str
        """
        Retrieve a valid link to the model file(s).
        If the model URL is a file system link, it will be returned directly.
        If the model URL is points to a remote location (http/s3/gs etc.),
        it will download the file(s) and return the temporary location of the downloaded model.

        :param bool extract_archive: If True and the model is of type 'packaged' (e.g. TensorFlow compressed folder)
            The returned path will be a temporary folder containing the archive content
        :param bool raise_on_error: If True and the artifact could not be downloaded,
            raise ValueError, otherwise return None on failure and output log warning.

        :return: A local path to the model (or a downloaded copy of it).
        """
        if extract_archive and self._is_package():
            return self.get_weights_package(return_path=True, raise_on_error=raise_on_error)
        return self.get_weights(raise_on_error=raise_on_error)

    def _get_base_model(self):
        if self._base_model:
            return self._base_model

        if not self._base_model_id:
            # this shouldn't actually happen
            raise Exception('Missing model ID, cannot create an empty model')
        self._base_model = _Model(
            upload_storage_uri=None,
            cache_dir=get_cache_dir(),
            model_id=self._base_model_id,
        )
        return self._base_model

    def _get_model_data(self):
        return self._get_base_model().data

    @classmethod
    def query_models(
            cls,
            project_name=None,  # type: Optional[str]
            model_name=None,  # type: Optional[str]
            tags=None,  # type: Optional[Sequence[str]]
            only_published=False,  # type: bool
            include_archived=False,  # type: bool
            max_results=None,  # type: Optional[int]
    ):
        # type: (...) -> List[Model]
        """
        Return Model objects from the project artifactory.
        Filter based on project-name / model-name / tags.
        List is always returned sorted by descending last update time (i.e. latest model is the first in the list)

        :param project_name: Optional, filter based project name string, if not given query models from all projects
        :param model_name: Optional Model name as shown in the model artifactory
        :param tags: Optional filter models based on list of tags, example: ['production', 'verified', '-qa']
            Notice use '-' prefix to filter out tags.
        :param only_published: If True only return published models.
        :param include_archived: If True return archived models.
        :param max_results: Optional return the last X models,
            sorted by last update time (from the most recent to the least).

        :return: ModeList of Models objects
        """
        if project_name:
            # noinspection PyProtectedMember
            res = _Model._get_default_session().send(
                projects.GetAllRequest(
                    name=exact_match_regex(project_name),
                    only_fields=['id', 'name', 'last_update']
                )
            )
            project = get_single_result(entity='project', query=project_name, results=res.response.projects)
        else:
            project = None

        only_fields = ['id', 'created', 'system_tags']

        # noinspection PyProtectedMember
        res = _Model._get_default_session().send(
            models.GetAllRequest(
                project=[project.id] if project else None,
                name=model_name or None,
                only_fields=only_fields,
                tags=tags or None,
                system_tags=["-" + cls._archived_tag] if not include_archived else None,
                ready=True if only_published else None,
                order_by=['-created'],
                page=0 if max_results else None,
                page_size=max_results or None,
            )
        )
        if not res.response.models:
            return []

        return [Model(model_id=m.id) for m in res.response.models]

    @property
    def id(self):
        # type: () -> str
        return self._base_model_id if self._base_model_id else super(Model, self).id

    @classmethod
    def remove(cls, model, delete_weights_file=True, force=False, raise_on_errors=False):
        # type: (Union[str, Model], bool, bool, bool) -> bool
        """
        Remove a model from the model repository.
        Optional, delete the model weights file from the remote storage.

        :param model: Model ID or Model object to remove
        :param delete_weights_file: If True (default) delete the weights file from the remote storage
        :param force: If True, remove model even if other Tasks are using this model. default False.
        :param raise_on_errors: If True, throw ValueError if something went wrong, default False.
        :return: True if Model was removed successfully
            partial removal returns False, i.e. Model was deleted but weights file deletion failed
        """
        if isinstance(model, str):
            model = Model(model_id=model)

        # noinspection PyBroadException
        try:
            weights_url = model.url
        except Exception:
            if raise_on_errors:
                raise ValueError("Could not find model id={}".format(model.id))
            return False

        try:
            # noinspection PyProtectedMember
            res = _Model._get_default_session().send(
                models.DeleteRequest(model.id, force=force),
            )
            response = res.wait()
            if not response.ok():
                if raise_on_errors:
                    raise ValueError("Could not remove model id={}: {}".format(model.id, response.meta))
                return False
        except SendError as ex:
            if raise_on_errors:
                raise ValueError("Could not remove model id={}: {}".format(model.id, ex))
            return False
        except ValueError:
            if raise_on_errors:
                raise
            return False
        except Exception as ex:
            if raise_on_errors:
                raise ValueError("Could not remove model id={}: {}".format(model.id, ex))
            return False

        if not delete_weights_file:
            return True

        helper = StorageHelper.get(url=weights_url)
        try:
            if not helper.delete(weights_url):
                if raise_on_errors:
                    raise ValueError("Could not remove model id={} weights file: {}".format(model.id, weights_url))
                return False
        except Exception as ex:
            if raise_on_errors:
                raise ValueError("Could not remove model id={} weights file \'{}\': {}".format(
                    model.id, weights_url, ex))
            return False

        return True


class InputModel(Model):
    """
    Load an existing model in the system, search by model id.
    The Model will be read-only and can be used to pre initialize a network
    We can connect the model to a task as input model, then when running remotely override it with the UI.
    """

    # noinspection PyProtectedMember
    _EMPTY_MODEL_ID = _Model._EMPTY_MODEL_ID

    @classmethod
    def import_model(
        cls,
        weights_url,  # type: str
        config_text=None,  # type: Optional[str]
        config_dict=None,  # type: Optional[dict]
        label_enumeration=None,  # type: Optional[Mapping[str, int]]
        name=None,  # type: Optional[str]
        tags=None,  # type: Optional[List[str]]
        comment=None,  # type: Optional[str]
        is_package=False,  # type: bool
        create_as_published=False,  # type: bool
        framework=None,  # type: Optional[str]
    ):
        # type: (...) -> InputModel
        """
        Create an InputModel object from a pre-trained model by specifying the URL of an initial weight files.
        Optionally, input a configuration, label enumeration, name for the model, tags describing the model,
        comment as a description of the model, indicate whether the model is a package, specify the model's
        framework, and indicate whether to immediately set the model's status to ``Published``.
        The model is read-only.

        The **ClearML Server** (backend) may already store the model's URL. If the input model's URL is not
        stored, meaning the model is new, then it is imported and ClearML stores its metadata.
        If the URL is already stored, the import process stops, ClearML issues a warning message, and ClearML
        reuses the model.

        In your Python experiment script, after importing the model, you can connect it to the main execution
        Task as an input model using :meth:`InputModel.connect` or :meth:`.Task.connect`. That initializes the
        network.

        .. note::
           Using the **ClearML Web-App** (user interface), you can reuse imported models and switch models in
           experiments.

        :param str weights_url: A valid URL for the initial weights file. If the **ClearML Web-App** (backend)
            already stores the metadata of a model with the same URL, that existing model is returned
            and ClearML ignores all other parameters.

            For example:

            - ``https://domain.com/file.bin``
            - ``s3://bucket/file.bin``
            - ``file:///home/user/file.bin``

        :param str config_text: The configuration as a string. This is usually the content of a configuration
            dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
        :type config_text: unconstrained text string
        :param dict config_dict: The configuration as a dictionary. Specify ``config_text`` or ``config_dict``,
            but not both.
        :param dict label_enumeration: Optional label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }
        :param str name: The name of the newly imported model. (Optional)
        :param tags: The list of tags which describe the model. (Optional)
        :type tags: list(str)
        :param str comment: A comment / description for the model. (Optional)
        :type comment: str
        :param is_package: Is the imported weights file is a package (Optional)

            - ``True`` - Is a package. Add a package tag to the model.
            - ``False`` - Is not a package. Do not add a package tag. (Default)

        :type is_package: bool
        :param bool create_as_published: Set the model's status to Published (Optional)

            - ``True`` - Set the status to Published.
            - ``False`` - Do not set the status to Published. The status will be Draft. (Default)

        :param str framework: The framework of the model. (Optional)
        :type framework: str or Framework object

        :return: The imported model or existing model (see above).
        """
        config_text = cls._resolve_config(config_text=config_text, config_dict=config_dict)
        weights_url = StorageHelper.conform_url(weights_url)
        if not weights_url:
            raise ValueError("Please provide a valid weights_url parameter")
        # convert local to file to remote one
        weights_url = CacheManager.get_remote_url(weights_url)

        extra = {'system_tags': ["-" + cls._archived_tag]} \
            if Session.check_min_api_version('2.3') else {'tags': ["-" + cls._archived_tag]}
        # noinspection PyProtectedMember
        result = _Model._get_default_session().send(models.GetAllRequest(
            uri=[weights_url],
            only_fields=["id", "name", "created"],
            **extra
        ))

        if result.response.models:
            logger = get_logger()

            logger.debug('A model with uri "{}" already exists. Selecting it'.format(weights_url))

            model = get_single_result(
                entity='model',
                query=weights_url,
                results=result.response.models,
                log=logger,
                raise_on_error=False,
            )

            logger.info("Selected model id: {}".format(model.id))

            return InputModel(model_id=model.id)

        base_model = _Model(
            upload_storage_uri=None,
            cache_dir=get_cache_dir(),
        )

        from .task import Task
        task = Task.current_task()
        if task:
            comment = 'Imported by task id: {}'.format(task.id) + ('\n' + comment if comment else '')
            project_id = task.project
            name = name or 'Imported by {}'.format(task.name or '')
            # do not register the Task, because we do not want it listed after as "output model",
            # the Task never actually created the Model
            task_id = None
        else:
            project_id = None
            task_id = None

        if not framework:
            # noinspection PyProtectedMember
            framework, file_ext = Framework._get_file_ext(
                framework=framework,
                filename=weights_url
            )

        base_model.update(
            design=config_text,
            labels=label_enumeration,
            name=name,
            comment=comment,
            tags=tags,
            uri=weights_url,
            framework=framework,
            project_id=project_id,
            task_id=task_id,
        )

        this_model = InputModel(model_id=base_model.id)
        this_model._base_model = base_model

        if is_package:
            this_model._set_package_tag()

        if create_as_published:
            this_model.publish()

        return this_model

    @classmethod
    def load_model(cls, weights_url, load_archived=False):
        # type: (str, bool) -> InputModel
        """
        Load an already registered model based on a pre-existing model file (link must be valid). If the url to the
        weights file already exists, the returned object is a Model representing the loaded Model. If no registered
        model with the specified url is found, ``None`` is returned.

        :param weights_url: The valid url for the weights file (string).

            Examples:

            .. code-block:: py

                "https://domain.com/file.bin" or "s3://bucket/file.bin" or "file:///home/user/file.bin".

            .. note::
                If a model with the exact same URL exists, it will be used, and all other arguments will be ignored.

        :param bool load_archived: Load archived models

            - ``True`` - Load the registered Model, if it is archived.
            - ``False`` - Ignore archive models.

        :return: The InputModel object, or None if no model could be found.
        """
        weights_url = StorageHelper.conform_url(weights_url)
        if not weights_url:
            raise ValueError("Please provide a valid weights_url parameter")

        # convert local to file to remote one
        weights_url = CacheManager.get_remote_url(weights_url)

        if not load_archived:
            # noinspection PyTypeChecker
            extra = {'system_tags': ["-" + _Task.archived_tag]} \
                if Session.check_min_api_version('2.3') else {'tags': ["-" + cls._archived_tag]}
        else:
            extra = {}

        # noinspection PyProtectedMember
        result = _Model._get_default_session().send(models.GetAllRequest(
            uri=[weights_url],
            only_fields=["id", "name", "created"],
            **extra
        ))

        if not result or not result.response or not result.response.models:
            return None

        logger = get_logger()
        model = get_single_result(
            entity='model',
            query=weights_url,
            results=result.response.models,
            log=logger,
            raise_on_error=False,
        )

        return InputModel(model_id=model.id)

    @classmethod
    def empty(cls, config_text=None, config_dict=None, label_enumeration=None):
        # type: (Optional[str], Optional[dict], Optional[Mapping[str, int]]) -> InputModel
        """
        Create an empty model object. Later, you can assign a model to the empty model object.

        :param config_text: The model configuration as a string. This is usually the content of a configuration
            dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
        :type config_text: unconstrained text string
        :param dict config_dict: The model configuration as a dictionary. Specify ``config_text`` or ``config_dict``,
            but not both.
        :param dict label_enumeration: The label enumeration dictionary of string (label) to integer (value) pairs.
            (Optional)

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        :return: An empty model object.
        """
        design = cls._resolve_config(config_text=config_text, config_dict=config_dict)

        this_model = InputModel(model_id=cls._EMPTY_MODEL_ID)
        this_model._base_model = m = _Model(
            cache_dir=None,
            upload_storage_uri=None,
            model_id=cls._EMPTY_MODEL_ID,
        )
        # noinspection PyProtectedMember
        m._data.design = _Model._wrap_design(design)
        # noinspection PyProtectedMember
        m._data.labels = label_enumeration
        return this_model

    def __init__(self, model_id=None, name=None, project=None, tags=None, only_published=False):
        # type: (Optional[str], Optional[str], Optional[str], Optional[Sequence[str]], bool) -> None
        """
        Load a model from the Model artifactory,
        based on model_id (uuid) or a model name/projects/tags combination.

        :param model_id: The ClearML Id (system UUID) of the input model whose metadata the **ClearML Server**
            (backend) stores. If provided all other arguments are ignored
        :param name: Model name to search and load
        :param project: Model project name to search model in
        :param tags: Model tags list to filter by
        :param only_published: If True filter out non-published (draft) models
        """
        if not model_id:
            found_models = self.query_models(
                project_name=project, model_name=name, tags=tags, only_published=only_published)
            if not found_models:
                raise ValueError("Could not locate model with project={} name={} tags={} published={}".format(
                    project, name, tags, only_published))
            model_id = found_models[0].id
        super(InputModel, self).__init__(model_id)

    @property
    def id(self):
        # type: () -> str
        return self._base_model_id

    def connect(self, task, name=None):
        # type: (Task, Optional[str]) -> None
        """
        Connect the current model to a Task object, if the model is preexisting. Preexisting models include:

        - Imported models (InputModel objects created using the :meth:`Logger.import_model` method).
        - Models whose metadata is already in the ClearML platform, meaning the InputModel object is instantiated
          from the ``InputModel`` class specifying the the model's ClearML Id as an argument.
        - Models whose origin is not ClearML that are used to create an InputModel object. For example,
          models created using TensorFlow models.

        When the experiment is executed remotely in a worker, the input model already specified in the experiment is
        used.

        .. note::
           The **ClearML Web-App** allows you to switch one input model for another and then enqueue the experiment
           to execute in a worker.

        :param object task: A Task object.
        :param str name: The model name to be stored on the Task
            (default the filename, of the model weights, without the file extension)
        """
        self._set_task(task)

        model_id = None
        # noinspection PyProtectedMember
        if running_remotely() and (task.is_main_task() or task._is_remote_main_task()):
            input_models = task.input_models_id
            try:
                model_id = next(m_id for m_name, m_id in input_models if m_name == (name or 'Input Model'))
                self._base_model_id = model_id
                self._base_model = InputModel(model_id=model_id)._get_base_model()
            except StopIteration:
                model_id = None

        if not model_id:
            # we should set the task input model to point to us
            model = self._get_base_model()
            # try to store the input model id, if it is not empty
            # (Empty Should not happen)
            if model.id != self._EMPTY_MODEL_ID:
                task.set_input_model(model_id=model.id, name=name)
            # only copy the model design if the task has no design to begin with
            # noinspection PyProtectedMember
            if not self._task._get_model_config_text() and model.model_design:
                # noinspection PyProtectedMember
                task._set_model_config(config_text=model.model_design)
            if not self._task.get_labels_enumeration() and model.data.labels:
                task.set_model_label_enumeration(model.data.labels)


class OutputModel(BaseModel):
    """
    Create an output model for a Task (experiment) to store the training results.

    The OutputModel object is always connected to a Task object, because it is instantiated with a Task object
    as an argument. It is, therefore, automatically registered as the Task's (experiment's) output model.

    The OutputModel object is read-write.

    A common use case is to reuse the OutputModel object, and override the weights after storing a model snapshot.
    Another use case is to create multiple OutputModel objects for a Task (experiment), and after a new high score
    is found, store a model snapshot.

    If the model configuration and / or the model's label enumeration
    are ``None``, then the output model is initialized with the values from the Task object's input model.

    .. note::
       When executing a Task (experiment) remotely in a worker, you can modify the model configuration and / or model's
       label enumeration using the **ClearML Web-App**.
    """

    _default_output_uri = None

    @property
    def published(self):
        # type: () -> bool
        """
        Get the published state of this model.

        :return:

        """
        if not self.id:
            return False
        return self._get_base_model().locked

    @property
    def config_text(self):
        # type: () -> str
        """
        Get the configuration as a string. For example, prototxt, an ini file, or Python code to evaluate.

        :return: The configuration.
        """
        # noinspection PyProtectedMember
        return _Model._unwrap_design(self._get_model_data().design)

    @config_text.setter
    def config_text(self, value):
        # type: (str) -> None
        """
        Set the configuration. Store a blob of text for custom usage.
        """
        self.update_design(config_text=value)

    @property
    def config_dict(self):
        # type: () -> dict
        """
        Get the configuration as a dictionary parsed from the ``config_text`` text. This usually represents the model
        configuration. For example, from prototxt to ini file or python code to evaluate.

        :return: The configuration.
        """
        return self._text_to_config_dict(self.config_text)

    @config_dict.setter
    def config_dict(self, value):
        # type: (dict) -> None
        """
        Set the configuration. Saved in the model object.

        :param dict value: The configuration parameters.
        """
        self.update_design(config_dict=value)

    @property
    def labels(self):
        # type: () -> Dict[str, int]
        """
        Get the label enumeration as a dictionary of string (label) to integer (value) pairs.

        For example:

        .. code-block:: javascript

           {
                'background': 0,
                'person': 1
           }

        :return: The label enumeration.
        """
        return self._get_model_data().labels

    @labels.setter
    def labels(self, value):
        # type: (Mapping[str, int]) -> None
        """
        Set the label enumeration.

        :param dict value: The label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        """
        self.update_labels(labels=value)

    @property
    def upload_storage_uri(self):
        # type: () -> str
        return self._get_base_model().upload_storage_uri

    def __init__(
        self,
        task=None,  # type: Optional[Task]
        config_text=None,  # type: Optional[str]
        config_dict=None,  # type: Optional[dict]
        label_enumeration=None,  # type: Optional[Mapping[str, int]]
        name=None,  # type: Optional[str]
        tags=None,  # type: Optional[List[str]]
        comment=None,  # type: Optional[str]
        framework=None,  # type: Optional[Union[str, Framework]]
        base_model_id=None,  # type: Optional[str]
    ):
        """
        Create a new model and immediately connect it to a task.

        We do not allow for Model creation without a task, so we always keep track on how we created the models
        In remote execution, Model parameters can be overridden by the Task
        (such as model configuration & label enumerator)

        :param task: The Task object with which the OutputModel object is associated.
        :type task: Task
        :param config_text: The configuration as a string. This is usually the content of a configuration
            dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
        :type config_text: unconstrained text string
        :param dict config_dict: The configuration as a dictionary.
            Specify ``config_dict`` or ``config_text``, but not both.
        :param dict label_enumeration: The label enumeration dictionary of string (label) to integer (value) pairs.
            (Optional)

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        :param str name: The name for the newly created model. (Optional)
        :param list(str) tags: A list of strings which are tags for the model. (Optional)
        :param str comment: A comment / description for the model. (Optional)
        :param framework: The framework of the model or a Framework object. (Optional)
        :type framework: str or Framework object
        :param base_model_id: optional, model id to be reused
        """
        if not task:
            from .task import Task
            task = Task.current_task()
            if not task:
                raise ValueError("task object was not provided, and no current task was found")

        super(OutputModel, self).__init__(task=task)

        config_text = self._resolve_config(config_text=config_text, config_dict=config_dict)

        self._model_local_filename = None
        self._last_uploaded_url = None
        self._base_model = None
        self._base_model_id = None
        self._task_connect_name = None
        # noinspection PyProtectedMember
        self._floating_data = create_dummy_model(
            design=_Model._wrap_design(config_text),
            labels=label_enumeration or task.get_labels_enumeration(),
            name=name or self._task.name,
            tags=tags,
            comment='{} by task id: {}'.format('Created' if not base_model_id else 'Overwritten', task.id) +
                    ('\n' + comment if comment else ''),
            framework=framework,
            upload_storage_uri=task.output_uri,
        )
        # If we have no real model ID, we are done
        if not base_model_id:
            return

        # noinspection PyBroadException
        try:
            # noinspection PyProtectedMember
            _base_model = self._task._get_output_model(model_id=base_model_id)
            _base_model.update(
                labels=self._floating_data.labels,
                design=self._floating_data.design,
                task_id=self._task.id,
                project_id=self._task.project,
                name=self._floating_data.name or self._task.name,
                comment=('{}\n{}'.format(_base_model.comment, self._floating_data.comment)
                         if (_base_model.comment and self._floating_data.comment and
                             self._floating_data.comment not in _base_model.comment)
                         else (_base_model.comment or self._floating_data.comment)),
                tags=self._floating_data.tags,
                framework=self._floating_data.framework,
                upload_storage_uri=self._floating_data.upload_storage_uri
            )
            self._base_model = _base_model
            self._floating_data = None
            name = self._task_connect_name or Path(_base_model.uri).stem
        except Exception:
            pass
        self.connect(task, name=name)

    def connect(self, task, name=None):
        # type: (Task, Optional[str]) -> None
        """
        Connect the current model to a Task object, if the model is a preexisting model. Preexisting models include:

        - Imported models.
        - Models whose metadata the **ClearML Server** (backend) is already storing.
        - Models from another source, such as frameworks like TensorFlow.

        :param object task: A Task object.
        :param str name: The model name as it would appear on the Task object.
            The model object itself can have a different name,
            this is designed to support multiple models used/created by a single Task.
            Use examples would be GANs or model ensemble
        """
        if self._task != task:
            raise ValueError('Can only connect preexisting model to task, but this is a fresh model')

        if name:
            self._task_connect_name = name

        # we should set the task input model to point to us
        model = self._get_base_model()

        # only copy the model design if the task has no design to begin with
        # noinspection PyProtectedMember
        if not self._task._get_model_config_text():
            # noinspection PyProtectedMember
            task._set_model_config(
                config_text=model.model_design if hasattr(model, "model_design") else model.design.get("design", "")
            )
        if not self._task.get_labels_enumeration():
            task.set_model_label_enumeration(model.data.labels if hasattr(model, "data") else model.labels)

        if self._base_model:
            self._base_model.update_for_task(
                task_id=self._task.id, model_id=self.id, type_="output", name=self._task_connect_name)

    def set_upload_destination(self, uri):
        # type: (str) -> None
        """
        Set the URI of the storage destination for uploaded model weight files.
        Supported storage destinations include S3, Google Cloud Storage), and file locations.

        Using this method, files uploads are separate and then a link to each is stored in the model object.

        .. note::
           For storage requiring credentials, the credentials are stored in the ClearML configuration file,
           ``~/clearml.conf``.

        :param str uri: The URI of the upload storage destination.

            For example:

            - ``s3://bucket/directory/``
            - ``file:///tmp/debug/``

        :return bool: The status of whether the storage destination schema is supported.

            - ``True`` - The storage destination scheme is supported.
            - ``False`` - The storage destination scheme is not supported.
        """
        if not uri:
            return

        # Test if we can update the model.
        self._validate_update()

        # Create the storage helper
        storage = StorageHelper.get(uri)

        # Verify that we can upload to this destination
        try:
            uri = storage.verify_upload(folder_uri=uri)
        except Exception:
            raise ValueError("Could not set destination uri to: %s [Check write permissions]" % uri)

        # store default uri
        self._get_base_model().upload_storage_uri = uri

    def update_weights(
        self,
        weights_filename=None,  # type: Optional[str]
        upload_uri=None,  # type: Optional[str]
        target_filename=None,  # type: Optional[str]
        auto_delete_file=True,  # type: bool
        register_uri=None,  # type: Optional[str]
        iteration=None,  # type: Optional[int]
        update_comment=True,  # type: bool
        is_package=False,  # type: bool
    ):
        # type: (...) -> str
        """
        Update the model weights from a locally stored model filename.

        .. note::
           Uploading the model is a background process. A call to this method returns immediately.

        :param str weights_filename: The name of the locally stored weights file to upload.
            Specify ``weights_filename`` or ``register_uri``, but not both.
        :param str upload_uri: The URI of the storage destination for model weights upload. The default value
            is the previously used URI. (Optional)
        :param str target_filename: The newly created filename in the storage destination location. The default value
            is the ``weights_filename`` value. (Optional)
        :param bool auto_delete_file: Delete the temporary file after uploading (Optional)

            - ``True`` - Delete (Default)
            - ``False`` - Do not delete

        :param str register_uri: The URI of an already uploaded weights file. The URI must be valid. Specify
            ``register_uri`` or ``weights_filename``, but not both.
        :param int iteration: The iteration number.
        :param bool update_comment: Update the model comment with the local weights file name (to maintain
            provenance) (Optional)

            - ``True`` - Update model comment (Default)
            - ``False`` - Do not update
        :param bool is_package: Mark the weights file as compressed package, usually a zip file.

        :return: The uploaded URI.
        """

        def delete_previous_weights_file(filename=weights_filename):
            try:
                if filename:
                    os.remove(filename)
            except OSError:
                self._log.debug('Failed removing temporary file %s' % filename)

        # test if we can update the model
        if self.id and self.published:
            raise ValueError('Model is published and cannot be changed')

        if (not weights_filename and not register_uri) or (weights_filename and register_uri):
            raise ValueError('Model update must have either local weights file to upload, '
                             'or pre-uploaded register_uri, never both')

        # only upload if we are connected to a task
        if not self._task:
            raise Exception('Missing a task for this model')

        if weights_filename is not None:
            # Check if weights_filename is a folder, is package upload
            if Path(weights_filename).is_dir():
                return self.update_weights_package(
                    weights_path=weights_filename,
                    upload_uri=upload_uri,
                    target_filename=target_filename or Path(weights_filename).name,
                    auto_delete_file=auto_delete_file,
                    iteration=iteration)

            # make sure we delete the previous file, if it exists
            if self._model_local_filename != weights_filename:
                delete_previous_weights_file(self._model_local_filename)
            # store temp filename for deletion next time, if needed
            if auto_delete_file:
                self._model_local_filename = weights_filename

        # make sure the created model is updated:
        out_model_file_name = target_filename or weights_filename or register_uri
        name = Path(out_model_file_name).stem if out_model_file_name else (self._task_connect_name or "Output Model")
        model = self._get_force_base_model(task_model_entry=name)
        if not model:
            raise ValueError('Failed creating internal output model')

        # select the correct file extension based on the framework,
        # or update the framework based on the file extension
        # noinspection PyProtectedMember
        framework, file_ext = Framework._get_file_ext(
            framework=self._get_model_data().framework,
            filename=target_filename or weights_filename or register_uri
        )

        if weights_filename:
            target_filename = target_filename or Path(weights_filename).name
            if not target_filename.lower().endswith(file_ext):
                target_filename += file_ext

        # set target uri for upload (if specified)
        if upload_uri:
            self.set_upload_destination(upload_uri)

        # let us know the iteration number, we put it in the comment section for now.
        if update_comment:
            comment = self.comment or ''
            iteration_msg = 'snapshot {} stored'.format(weights_filename or register_uri)
            if not comment.startswith('\n'):
                comment = '\n' + comment
            comment = iteration_msg + comment
        else:
            comment = None

        # if we have no output destination, just register the local model file
        if weights_filename and not self.upload_storage_uri and not self._task.storage_uri:
            register_uri = weights_filename
            weights_filename = None
            auto_delete_file = False
            self._log.info('No output storage destination defined, registering local model %s' % register_uri)

        # start the upload
        if weights_filename:
            if not model.upload_storage_uri:
                self.set_upload_destination(self.upload_storage_uri or self._task.storage_uri)

            output_uri = model.update_and_upload(
                model_file=weights_filename,
                task_id=self._task.id,
                async_enable=True,
                target_filename=target_filename,
                framework=self.framework or framework,
                comment=comment,
                cb=delete_previous_weights_file if auto_delete_file else None,
                iteration=iteration or self._task.get_last_iteration(),
            )
        elif register_uri:
            register_uri = StorageHelper.conform_url(register_uri)
            output_uri = model.update(uri=register_uri, task_id=self._task.id, framework=framework, comment=comment)
        else:
            output_uri = None

        self._last_uploaded_url = output_uri

        if is_package:
            self._set_package_tag()

        return output_uri

    def update_weights_package(
        self,
        weights_filenames=None,  # type: Optional[Sequence[str]]
        weights_path=None,  # type: Optional[str]
        upload_uri=None,  # type: Optional[str]
        target_filename=None,  # type: Optional[str]
        auto_delete_file=True,  # type: bool
        iteration=None  # type: Optional[int]
    ):
        # type: (...) -> str
        """
        Update the model weights from locally stored model files, or from directory containing multiple files.

        .. note::
           Uploading the model weights is a background process. A call to this method returns immediately.

        :param weights_filenames: The file names of the locally stored model files. Specify ``weights_filenames``,
            or ``weights_path``, but not both.
        :type weights_filenames: list(str)
        :param weights_path: The directory path to a package. All the files in the directory will be uploaded.
            Specify ``weights_path`` or ``weights_filenames``, but not both.
        :type weights_path: str
        :param str upload_uri: The URI of the storage destination for the model weights upload. The default
            is the previously used URI. (Optional)
        :param str target_filename: The newly created filename in the storage destination URI location. The default
            is the value specified in the ``weights_filename`` parameter.  (Optional)
        :param bool auto_delete_file: Delete temporary file after uploading  (Optional)

            - ``True`` - Delete (Default)
            - ``False`` - Do not delete

        :param int iteration: The iteration number.

        :return: The uploaded URI for the weights package.
        """
        # create list of files
        if (not weights_filenames and not weights_path) or (weights_filenames and weights_path):
            raise ValueError('Model update weights package should get either '
                             'directory path to pack or a list of files')

        if not weights_filenames:
            weights_filenames = list(map(six.text_type, Path(weights_path).rglob('*')))
        elif weights_filenames and len(weights_filenames) > 1:
            weights_path = get_common_path(weights_filenames)

        # create packed model from all the files
        fd, zip_file = mkstemp(prefix='model_package.', suffix='.zip')
        try:
            with zipfile.ZipFile(zip_file, 'w', allowZip64=True, compression=zipfile.ZIP_STORED) as zf:
                for filename in weights_filenames:
                    relative_file_name = Path(filename).name if not weights_path else \
                        Path(filename).absolute().relative_to(Path(weights_path).absolute()).as_posix()
                    zf.write(filename, arcname=relative_file_name)
        finally:
            os.close(fd)

        # now we can delete the files (or path if provided)
        if auto_delete_file:
            def safe_remove(path, is_dir=False):
                try:
                    (os.rmdir if is_dir else os.remove)(path)
                except OSError:
                    self._log.info('Failed removing temporary {}'.format(path))

            for filename in weights_filenames:
                safe_remove(filename)
            if weights_path:
                safe_remove(weights_path, is_dir=True)

        if target_filename and not target_filename.lower().endswith('.zip'):
            target_filename += '.zip'

        # and now we should upload the file, always delete the temporary zip file
        comment = self.comment or ''
        iteration_msg = 'snapshot {} stored'.format(str(weights_filenames))
        if not comment.startswith('\n'):
            comment = '\n' + comment
        comment = iteration_msg + comment
        self.comment = comment
        uploaded_uri = self.update_weights(weights_filename=zip_file, auto_delete_file=True, upload_uri=upload_uri,
                                           target_filename=target_filename or 'model_package.zip',
                                           iteration=iteration, update_comment=False)
        # set the model tag (by now we should have a model object) so we know we have packaged file
        self._set_package_tag()
        return uploaded_uri

    def update_design(self, config_text=None, config_dict=None):
        # type: (Optional[str], Optional[dict]) -> bool
        """
        Update the model configuration. Store a blob of text for custom usage.

        .. note::
           This method's behavior is lazy. The design update is only forced when the weights
           are updated.

        :param config_text: The configuration as a string. This is usually the content of a configuration
            dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
        :type config_text: unconstrained text string
        :param dict config_dict: The configuration as a dictionary. Specify ``config_text`` or ``config_dict``,
            but not both.

        :return: True, update successful. False, update not successful.
        """
        if not self._validate_update():
            return False

        config_text = self._resolve_config(config_text=config_text, config_dict=config_dict)

        if self._task and not self._task.get_model_config_text():
            self._task.set_model_config(config_text=config_text)

        if self.id:
            # update the model object (this will happen if we resumed a training task)
            result = self._get_force_base_model().edit(design=config_text)
        else:
            # noinspection PyProtectedMember
            self._floating_data.design = _Model._wrap_design(config_text)
            result = Waitable()

        # you can wait on this object
        return result

    def update_labels(self, labels):
        # type: (Mapping[str, int]) -> Optional[Waitable]
        """
        Update the label enumeration.

        :param dict labels: The label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        :return:
        """
        validate_dict(labels, key_types=six.string_types, value_types=six.integer_types, desc='label enumeration')

        if not self._validate_update():
            return

        if self._task:
            self._task.set_model_label_enumeration(labels)

        if self.id:
            # update the model object (this will happen if we resumed a training task)
            result = self._get_force_base_model().edit(labels=labels)
        else:
            self._floating_data.labels = labels
            result = Waitable()

        # you can wait on this object
        return result

    @classmethod
    def wait_for_uploads(cls, timeout=None, max_num_uploads=None):
        # type: (Optional[float], Optional[int]) -> None
        """
        Wait for any pending or in-progress model uploads to complete. If no uploads are pending or in-progress,
        then the ``wait_for_uploads`` returns immediately.

        :param float timeout: The timeout interval to wait for uploads (seconds). (Optional).
        :param int max_num_uploads: The maximum number of uploads to wait for. (Optional).
        """
        _Model.wait_for_results(timeout=timeout, max_num_uploads=max_num_uploads)

    @classmethod
    def set_default_upload_uri(cls, output_uri):
        # type: (Optional[str]) -> None
        """
        Set the default upload uri for all OutputModels

        :param output_uri: URL for uploading models. examples:
            https://demofiles.demo.clear.ml, s3://bucket/, gs://bucket/, azure://bucket/, file:///mnt/shared/nfs
        """
        cls._default_output_uri = str(output_uri) if output_uri else None

    def _get_force_base_model(self, model_name=None, task_model_entry=None):
        if self._base_model:
            return self._base_model

        # create a new model from the task
        # noinspection PyProtectedMember
        self._base_model = self._task._get_output_model(model_id=None)
        # update the model from the task inputs
        labels = self._task.get_labels_enumeration()
        # noinspection PyProtectedMember
        config_text = self._task._get_model_config_text()
        model_name = model_name or self._floating_data.name or self._task.name
        task_model_entry = task_model_entry or self._task_connect_name or Path(self._get_model_data().uri).stem
        parent = self._task.input_models_id.get(task_model_entry)
        self._base_model.update(
            labels=self._floating_data.labels or labels,
            design=self._floating_data.design or config_text,
            task_id=self._task.id,
            project_id=self._task.project,
            parent_id=parent,
            name=model_name,
            comment=self._floating_data.comment,
            tags=self._floating_data.tags,
            framework=self._floating_data.framework,
            upload_storage_uri=self._floating_data.upload_storage_uri
        )

        # remove model floating change set, by now they should have matched the task.
        self._floating_data = None

        # now we have to update the creator task so it points to us
        if str(self._task.status) not in (
                str(self._task.TaskStatusEnum.created), str(self._task.TaskStatusEnum.in_progress)):
            self._log.warning('Could not update last created model in Task {}, '
                              'Task status \'{}\' cannot be updated'.format(self._task.id, self._task.status))
        else:
            self._base_model.update_for_task(
                task_id=self._task.id, model_id=self.id, type_="output", name=task_model_entry)

        return self._base_model

    def _get_base_model(self):
        if self._floating_data:
            return self._floating_data
        return self._get_force_base_model()

    def _get_model_data(self):
        if self._base_model:
            return self._base_model.data
        return self._floating_data

    def _validate_update(self):
        # test if we can update the model
        if self.id and self.published:
            raise ValueError('Model is published and cannot be changed')

        return True

    def _get_last_uploaded_filename(self):
        if not self._last_uploaded_url and not self.url:
            return None
        return Path(self._last_uploaded_url or self.url).name


class Waitable(object):
    def wait(self, *_, **__):
        return True


# clearml/task.py

import atexit
import json
import os
import shutil
import signal
import sys
import threading
import time
from argparse import ArgumentParser
from logging import getLogger
from operator import attrgetter
from tempfile import mkstemp, mkdtemp
from zipfile import ZipFile, ZIP_DEFLATED

try:
    # noinspection PyCompatibility
    from collections.abc import Sequence as CollectionsSequence
except ImportError:
    from collections import Sequence as CollectionsSequence

from typing import Optional, Union, Mapping, Sequence, Any, Dict, Iterable, TYPE_CHECKING, Callable, Tuple, List

import psutil
import six
from pathlib2 import Path

from .backend_config.defs import get_active_config_file, get_config_file
from .backend_api.services import tasks, projects
from .backend_api.session.session import (
    Session, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_HOST, ENV_WEB_HOST, ENV_FILES_HOST, )
from .backend_interface.metrics import Metrics
from .backend_interface.model import Model as BackendModel
from .backend_interface.task import Task as _Task
from .backend_interface.task.log import TaskHandler
from .backend_interface.task.development.worker import DevWorker
from .backend_interface.task.repo import ScriptInfo
from .backend_interface.task.models import TaskModels
from .backend_interface.util import get_single_result, exact_match_regex, make_message, mutually_exclusive, get_queue_id
from .binding.absl_bind import PatchAbsl
from .binding.artifacts import Artifacts, Artifact
from .binding.environ_bind import EnvironmentBind, PatchOsFork
from .binding.frameworks.fastai_bind import PatchFastai
from .binding.frameworks.lightgbm_bind import PatchLIGHTgbmModelIO
from .binding.frameworks.pytorch_bind import PatchPyTorchModelIO
from .binding.frameworks.tensorflow_bind import TensorflowBinding
from .binding.frameworks.xgboost_bind import PatchXGBoostModelIO
from .binding.frameworks.catboost_bind import PatchCatBoostModelIO
from .binding.frameworks.megengine_bind import PatchMegEngineModelIO
from .binding.joblib_bind import PatchedJoblib
from .binding.matplotlib_bind import PatchedMatplotlib
from .binding.hydra_bind import PatchHydra
from .binding.click_bind import PatchClick
from .binding.jsonargs_bind import PatchJsonArgParse
from .config import (
    config, DEV_TASK_NO_REUSE, get_is_master_node, DEBUG_SIMULATE_REMOTE_TASK, DEV_DEFAULT_OUTPUT_URI,
    deferred_config, TASK_SET_ITERATION_OFFSET, )
from .config import running_remotely, get_remote_task_id
from .config.cache import SessionCache
from .debugging.log import LoggerRoot
from .errors import UsageError
from .logger import Logger
from .model import Model, InputModel, OutputModel
from .task_parameters import TaskParameters
from .utilities.config import verify_basic_value
from .binding.args import (
    argparser_parseargs_called, get_argparser_last_args,
    argparser_update_currenttask, )
from .utilities.dicts import ReadOnlyDict, merge_dicts
from .utilities.proxy_object import (
    ProxyDictPreWrite, ProxyDictPostWrite, flatten_dictionary,
    nested_from_flat_dictionary, naive_nested_from_flat_dictionary, )
from .utilities.resource_monitor import ResourceMonitor
from .utilities.seed import make_deterministic
from .utilities.lowlevel.threads import get_current_thread_id
from .utilities.process.mp import BackgroundMonitor, leave_process
# noinspection PyProtectedMember
from .backend_interface.task.args import _Arguments

if TYPE_CHECKING:
    import pandas
    import numpy
    from PIL import Image


class Task(_Task):
    """
    The ``Task`` class is a code template for a Task object which, together with its connected experiment components,
    represents the current running experiment. These connected components include hyperparameters, loggers,
    configuration, label enumeration, models, and other artifacts.

    The term "main execution Task" refers to the Task context for current running experiment. Python experiment scripts
    can create one, and only one, main execution Task. It is a traceable, and after a script runs and ClearML stores
    the Task in the **ClearML Server** (backend), it is modifiable, reproducible, executable by a worker, and you
    can duplicate it for further experimentation.

    The ``Task`` class and its methods allow you to create and manage experiments, as well as perform
    advanced experimentation functions, such as autoML.

    .. warning::
        Do not construct Task objects directly. Use one of the methods listed below to create experiments or
        reference existing experiments.

    For detailed information about creating Task objects, see the following methods:

    - Create a new reproducible Task - :meth:`Task.init`

    .. important::
        In some cases, ``Task.init`` may return a Task object which is already stored in **ClearML Server** (already
        initialized), instead of creating a new Task. For a detailed explanation of those cases, see the ``Task.init``
        method.

    - Manually create a new Task (no auto-logging will apply) - :meth:`Task.create`
    - Get the current running Task - :meth:`Task.current_task`
    - Get another (different) Task - :meth:`Task.get_task`

    .. note::
        The **ClearML** documentation often refers to a Task as, "Task (experiment)".

        "Task" refers to the class in the ClearML Python Client Package, the object in your Python experiment script,
        and the entity with which **ClearML Server** and **ClearML Agent** work.

        "Experiment" refers to your deep learning solution, including its connected components, inputs, and outputs,
        and is the experiment you can view, analyze, compare, modify, duplicate, and manage using the ClearML
        **Web-App** (UI).

        Therefore, a "Task" is effectively an "experiment", and "Task (experiment)" encompasses its usage throughout
        the ClearML.

        The exception to this Task behavior is sub-tasks (non-reproducible Tasks), which do not use the main execution
        Task. Creating a sub-task always creates a new Task with a new  Task ID.
    """

    TaskTypes = _Task.TaskTypes

    NotSet = object()

    __create_protection = object()
    __main_task = None  # type: Optional[Task]
    __exit_hook = None
    __forked_proc_main_pid = None
    __task_id_reuse_time_window_in_hours = deferred_config('development.task_reuse_time_window_in_hours', 24.0, float)
    __detect_repo_async = deferred_config('development.vcs_repo_detect_async', False)
    __default_output_uri = DEV_DEFAULT_OUTPUT_URI.get() or deferred_config('development.default_output_uri', None)

    class _ConnectedParametersType(object):
        argparse = "argument_parser"
        dictionary = "dictionary"
        task_parameters = "task_parameters"

        @classmethod
        def _options(cls):
            return {
                var for var, val in vars(cls).items()
                if isinstance(val, six.string_types)
            }

    def __init__(self, private=None, **kwargs):
        """
        .. warning::
            **Do not construct Task manually!**
            Please use :meth:`Task.init` or :meth:`Task.get_task`
        """
        if private is not Task.__create_protection:
            raise UsageError(
                'Task object cannot be instantiated externally, use Task.current_task() or Task.get_task(...)')
        self._repo_detect_lock = threading.RLock()

        super(Task, self).__init__(**kwargs)
        self._arguments = _Arguments(self)
        self._logger = None
        self._connected_output_model = None
        self._dev_worker = None
        self._connected_parameter_type = None
        self._detect_repo_async_thread = None
        self._resource_monitor = None
        self._calling_filename = None
        self._remote_functions_generated = {}
        # register atexit, so that we mark the task as stopped
        self._at_exit_called = False

    @classmethod
    def current_task(cls):
        # type: () -> Task
        """
        Get the current running Task (experiment). This is the main execution Task (task context) returned as a Task
        object.

        :return: The current running Task (experiment).
        """
        # check if we have no main Task, but the main process created one.
        if not cls.__main_task and cls.__get_master_id_task_id():
            # initialize the Task, connect to stdout
            cls.init()
        # return main Task
        return cls.__main_task

    @classmethod
    def init(
            cls,
            project_name=None,  # type: Optional[str]
            task_name=None,  # type: Optional[str]
            task_type=TaskTypes.training,  # type: Task.TaskTypes
            tags=None,  # type: Optional[Sequence[str]]
            reuse_last_task_id=True,  # type: Union[bool, str]
            continue_last_task=False,  # type: Union[bool, str, int]
            output_uri=None,  # type: Optional[Union[str, bool]]
            auto_connect_arg_parser=True,  # type: Union[bool, Mapping[str, bool]]
            auto_connect_frameworks=True,  # type: Union[bool, Mapping[str, bool]]
            auto_resource_monitoring=True,  # type: bool
            auto_connect_streams=True,  # type: Union[bool, Mapping[str, bool]]
    ):
        # type: (...) -> "Task"
        """
        Creates a new Task (experiment) if:

        - The Task never ran before. No Task with the same ``task_name`` and ``project_name`` is stored in
          **ClearML Server**.
        - The Task has run before (the same ``task_name`` and ``project_name``), and (a) it stored models and / or
          artifacts, or (b) its status is Published , or (c) it is Archived.
        - A new Task is forced by calling ``Task.init`` with ``reuse_last_task_id=False``.

        Otherwise, the already initialized Task object for the same ``task_name`` and ``project_name`` is returned.

        .. note::
            To reference another Task, instead of initializing the same Task more than once, call
            :meth:`Task.get_task`. For example, to "share" the same experiment in more than one script,
            call ``Task.get_task``. See the ``Task.get_task`` method for an example.

        For example:
        The first time the following code runs, it will create a new Task. The status will be Completed.

        .. code-block:: py

            from clearml import Task
            task = Task.init('myProject', 'myTask')

        If this code runs again, it will not create a new Task. It does not store a model or artifact,
        it is not Published (its status Completed) , it was not Archived, and a new Task is not forced.

        If the Task is Published or Archived, and run again, it will create a new Task with a new Task ID.

        The following code will create a new Task every time it runs, because it stores an artifact.

        .. code-block:: py

            task = Task.init('myProject', 'myOtherTask')

            d = {'a': '1'}
            task.upload_artifact('myArtifact', d)

        :param str project_name: The name of the project in which the experiment will be created. If the project does
            not exist, it is created. If ``project_name`` is ``None``, the repository name is used. (Optional)
        :param str task_name: The name of Task (experiment). If ``task_name`` is ``None``, the Python experiment
            script's file name is used. (Optional)
        :param TaskTypes task_type: The task type.

            Valid task types:

            - ``TaskTypes.training`` (default)
            - ``TaskTypes.testing``
            - ``TaskTypes.inference``
            - ``TaskTypes.data_processing``
            - ``TaskTypes.application``
            - ``TaskTypes.monitor``
            - ``TaskTypes.controller``
            - ``TaskTypes.optimizer``
            - ``TaskTypes.service``
            - ``TaskTypes.qc``
            - ``TaskTypes.custom``

        :param tags: Add a list of tags (str) to the created Task. For example: tags=['512x512', 'yolov3']
        :param bool reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID,
            and the same project and Task name.

            .. note::
               If the previously executed Task has artifacts or models, it will not be reused (overwritten)
               and a new Task will be created.
               When a Task is reused, the previous execution outputs are deleted, including console outputs and logs.

            The values are:

            - ``True`` - Reuse the last  Task ID. (default)
            - ``False`` - Force a new Task (experiment).
            - A string - You can also specify a Task ID (string) to be reused,
                instead of the cached ID based on the project/name combination.

        :param bool continue_last_task: Continue the execution of a previously executed Task (experiment)

            .. note::
                When continuing the executing of a previously executed Task,
                all previous artifacts / models/ logs are intact.
                New logs will continue iteration/step based on the previous-execution maximum iteration value.
                For example:
                The last train/loss scalar reported was iteration 100, the next report will be iteration 101.

            The values are:

            - ``True`` - Continue the the last Task ID.
                specified explicitly by reuse_last_task_id or implicitly with the same logic as reuse_last_task_id
            - ``False`` - Overwrite the execution of previous Task  (default).
            - A string - You can also specify a Task ID (string) to be continued.
                This is equivalent to `continue_last_task=True` and `reuse_last_task_id=a_task_id_string`.
            - An integer - Specify initial iteration offset (override the auto automatic last_iteration_offset)
                Pass 0, to disable the automatic last_iteration_offset or specify a different initial offset
                You can specify a Task ID to be used with `reuse_last_task_id='task_id_here'`

        :param str output_uri: The default location for output models and other artifacts.
            If True is passed, the default files_server will be used for model storage.
            In the default location, ClearML creates a subfolder for the output.
            The subfolder structure is the following:
            <output destination name> / <project name> / <task name>.<Task ID>

            The following are examples of ``output_uri`` values for the supported locations:

            - A shared folder: ``/mnt/share/folder``
            - S3: ``s3://bucket/folder``
            - Google Cloud Storage: ``gs://bucket-name/folder``
            - Azure Storage: ``azure://company.blob.core.windows.net/folder/``
            - Default file server: True

            .. important::

               For cloud storage, you must install the **ClearML** package for your cloud storage type,
               and then configure your storage credentials. For detailed information, see
               `ClearML Python Client Extras <./references/clearml_extras_storage/>`_ in the "ClearML Python Client
               Reference" section.

        :param auto_connect_arg_parser: Automatically connect an argparse object to the Task

            The values are:

            - ``True`` - Automatically connect. (default)
            - ``False`` - Do not automatically connect.
            - A dictionary - In addition to a boolean, you can use a dictionary for fined grained control of connected
                arguments. The dictionary keys are argparse variable names and the values are booleans.
                The ``False`` value excludes the specified argument from the Task's parameter section.
                Keys missing from the dictionary default to ``True``, you can change it to be ``False`` by adding
                ``*`` key as ``False`` to the dictionary.
                An empty dictionary defaults to ``False``.

            For example:

            .. code-block:: py

               auto_connect_arg_parser={'do_not_include_me': False, }

            .. code-block:: py

               auto_connect_arg_parser={"only_include_me": True, "*": False}

            .. note::
               To manually connect an argparse, use :meth:`Task.connect`.

        :param auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, XGBoost,
            scikit-learn, Keras callbacks, and TensorBoard/X to serialize plots, graphs, and the model location to
            the **ClearML Server** (backend), in addition to original output destination.

            The values are:

            - ``True`` - Automatically connect (default)
            - ``False`` - Do not automatically connect
            - A dictionary - In addition to a boolean, you can use a dictionary for fined grained control of connected
                frameworks. The dictionary keys are frameworks and the values are booleans.
                Keys missing from the dictionary default to ``True``, and an empty dictionary defaults to ``False``.

            For example:

            .. code-block:: py

               auto_connect_frameworks={
                   'matplotlib': True, 'tensorflow': True, 'tensorboard': True, 'pytorch': True,
                   'xgboost': True, 'scikit': True, 'fastai': True, 'lightgbm': True,
                   'hydra': True, 'detect_repository': True, 'tfdefines': True, 'joblib': True,
                   'megengine': True, 'jsonargparse': True, 'catboost': True
               }

        :param bool auto_resource_monitoring: Automatically create machine resource monitoring plots
            These plots appear in in the **ClearML Web-App (UI)**, **RESULTS** tab, **SCALARS** sub-tab,
            with a title of **:resource monitor:**.

            The values are:

            - ``True`` - Automatically create resource monitoring plots. (default)
            - ``False`` - Do not automatically create.
            - Class Type - Create ResourceMonitor object of the specified class type.

        :param auto_connect_streams: Control the automatic logging of stdout and stderr

            The values are:

            - ``True`` - Automatically connect (default)
            -  ``False`` - Do not automatically connect
            - A dictionary - In addition to a boolean, you can use a dictionary for fined grained control of stdout and
                stderr. The dictionary keys are 'stdout' , 'stderr' and 'logging', the values are booleans.
                Keys missing from the dictionary default to ``False``, and an empty dictionary defaults to ``False``.
                Notice, the default behaviour is logging stdout/stderr the
                `logging` module is logged as a by product of the stderr logging

            For example:

            .. code-block:: py

               auto_connect_streams={'stdout': True, 'stderr': True, 'logging': False}

        :return: The main execution Task (Task context).
        """

        def verify_defaults_match():
            validate = [
                ('project name', project_name, cls.__main_task.get_project_name()),
                ('task name', task_name, cls.__main_task.name),
                ('task type', str(task_type) if task_type else task_type, str(cls.__main_task.task_type)),
            ]

            for field, default, current in validate:
                if default is not None and default != current:
                    raise UsageError(
                        "Current task already created "
                        "and requested {field} '{default}' does not match current {field} '{current}'. "
                        "If you wish to create additional tasks use `Task.create`, "
                        "or close the current task with `task.close()` before calling `Task.init(...)`".format(
                            field=field,
                            default=default,
                            current=current,
                        )
                    )

        if cls.__main_task is not None:
            # if this is a subprocess, regardless of what the init was called for,
            # we have to fix the main task hooks and stdout bindings
            if cls.__forked_proc_main_pid != os.getpid() and cls.__is_subprocess():
                if task_type is None:
                    task_type = cls.__main_task.task_type
                # make sure we only do it once per process
                cls.__forked_proc_main_pid = os.getpid()
                # make sure we do not wait for the repo detect thread
                cls.__main_task._detect_repo_async_thread = None
                cls.__main_task._dev_worker = None
                cls.__main_task._resource_monitor = None
                # remove the logger from the previous process
                cls.__main_task.get_logger()
                # create a new logger (to catch stdout/err)
                cls.__main_task._logger = None
                cls.__main_task.__reporter = None
                # noinspection PyProtectedMember
                cls.__main_task._get_logger(auto_connect_streams=auto_connect_streams)
                cls.__main_task._artifacts_manager = Artifacts(cls.__main_task)
                # unregister signal hooks, they cause subprocess to hang
                # noinspection PyProtectedMember
                cls.__main_task.__register_at_exit(cls.__main_task._at_exit)
                # TODO: Check if the signal handler method is safe enough, for the time being, do not unhook
                # cls.__main_task.__register_at_exit(None, only_remove_signal_and_exception_hooks=True)

                # start all reporting threads
                BackgroundMonitor.start_all(task=cls.__main_task)

            if not running_remotely():
                verify_defaults_match()

            return cls.__main_task

        is_sub_process_task_id = None
        # check that we are not a child process, in that case do nothing.
        # we should not get here unless this is Windows/macOS platform, linux support fork
        if cls.__is_subprocess():
            class _TaskStub(object):
                def __call__(self, *args, **kwargs):
                    return self

                def __getattr__(self, attr):
                    return self

                def __setattr__(self, attr, val):
                    pass

            is_sub_process_task_id = cls.__get_master_id_task_id()
            # we could not find a task ID, revert to old stub behaviour
            if not is_sub_process_task_id:
                return _TaskStub()  # noqa

        elif running_remotely() and not get_is_master_node():
            # make sure we only do it once per process
            cls.__forked_proc_main_pid = os.getpid()
            # make sure everyone understands we should act as if we are a subprocess (fake pid 1)
            cls.__update_master_pid_task(pid=1, task=get_remote_task_id())
        else:
            # set us as master process (without task ID)
            cls.__update_master_pid_task()
            is_sub_process_task_id = None

        if task_type is None:
            # Backwards compatibility: if called from Task.current_task and task_type
            # was not specified, keep legacy default value of TaskTypes.training
            task_type = cls.TaskTypes.training
        elif isinstance(task_type, six.string_types):
            if task_type not in Task.TaskTypes.__members__:
                raise ValueError("Task type '{}' not supported, options are: {}".format(
                    task_type, Task.TaskTypes.__members__.keys()))
            task_type = Task.TaskTypes.__members__[str(task_type)]

        try:
            if not running_remotely():
                # if this is the main process, create the task
                if not is_sub_process_task_id:
                    task = cls._create_dev_task(
                        default_project_name=project_name,
                        default_task_name=task_name,
                        default_task_type=task_type,
                        tags=tags,
                        reuse_last_task_id=reuse_last_task_id,
                        continue_last_task=continue_last_task,
                        detect_repo=False if (
                                isinstance(auto_connect_frameworks, dict) and
                                not auto_connect_frameworks.get('detect_repository', True)) else True,
                        auto_connect_streams=auto_connect_streams,
                    )
                    # set defaults
                    if cls._offline_mode:
                        task.output_uri = None
                    elif output_uri:
                        task.output_uri = output_uri
                    elif cls.__default_output_uri:
                        task.output_uri = cls.__default_output_uri
                    # store new task ID
                    cls.__update_master_pid_task(task=task)
                else:
                    # subprocess should get back the task info
                    task = cls.get_task(task_id=is_sub_process_task_id)
            else:
                # if this is the main process, create the task
                if not is_sub_process_task_id:
                    task = cls(
                        private=cls.__create_protection,
                        task_id=get_remote_task_id(),
                        log_to_backend=False,
                    )
                    if cls.__default_output_uri and not task.output_uri:
                        task.output_uri = cls.__default_output_uri
                    # store new task ID
                    cls.__update_master_pid_task(task=task)
                    # make sure we are started
                    task.started(ignore_errors=True)
                    # continue last iteration if we had any
                    if task.data.last_iteration:
                        task.set_initial_iteration(int(task.data.last_iteration) + 1)
                else:
                    # subprocess should get back the task info
                    task = cls.get_task(task_id=is_sub_process_task_id)
        except Exception:
            raise
        else:
            Task.__main_task = task
            # register the main task for at exit hooks (there should only be one)
            task.__register_at_exit(task._at_exit)
            # always patch OS forking because of ProcessPool and the alike
            PatchOsFork.patch_fork()
            if auto_connect_frameworks:
                is_auto_connect_frameworks_bool = not isinstance(auto_connect_frameworks, dict)
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('hydra', True):
                    PatchHydra.update_current_task(task)
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('jsonargparse', True):
                    PatchJsonArgParse.update_current_task(task)
                if is_auto_connect_frameworks_bool or (
                        auto_connect_frameworks.get('scikit', True) and
                        auto_connect_frameworks.get('joblib', True)):
                    PatchedJoblib.update_current_task(task)
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('matplotlib', True):
                    PatchedMatplotlib.update_current_task(Task.__main_task)
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('tensorflow', True) \
                        or auto_connect_frameworks.get('tensorboard', True):
                    # allow to disable tfdefines
                    if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('tfdefines', True):
                        PatchAbsl.update_current_task(Task.__main_task)
                    TensorflowBinding.update_current_task(
                        task,
                        patch_reporting=(is_auto_connect_frameworks_bool
                                         or auto_connect_frameworks.get('tensorboard', True)),
                        patch_model_io=(is_auto_connect_frameworks_bool
                                        or auto_connect_frameworks.get('tensorflow', True)),
                    )
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('pytorch', True):
                    PatchPyTorchModelIO.update_current_task(task)
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('megengine', True):
                    PatchMegEngineModelIO.update_current_task(task)
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('xgboost', True):
                    PatchXGBoostModelIO.update_current_task(task)
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('catboost', True):
                    PatchCatBoostModelIO.update_current_task(task)
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('fastai', True):
                    PatchFastai.update_current_task(task)
                if is_auto_connect_frameworks_bool or auto_connect_frameworks.get('lightgbm', True):
                    PatchLIGHTgbmModelIO.update_current_task(task)
            if auto_resource_monitoring and not is_sub_process_task_id:
                resource_monitor_cls = auto_resource_monitoring \
                    if isinstance(auto_resource_monitoring, six.class_types) else ResourceMonitor
                task._resource_monitor = resource_monitor_cls(
                    task, report_mem_used_per_process=not config.get(
                        'development.worker.report_global_mem_used', False))
                task._resource_monitor.start()

            # make sure all random generators are initialized with new seed
            make_deterministic(task.get_random_seed())

            if auto_connect_arg_parser:
                EnvironmentBind.update_current_task(Task.__main_task)

                # Patch ArgParser to be aware of the current task
                argparser_update_currenttask(Task.__main_task)
                # Patch Click
                PatchClick.patch(Task.__main_task)

                # set excluded arguments
                if isinstance(auto_connect_arg_parser, dict):
                    task._arguments.exclude_parser_args(auto_connect_arg_parser)

                # Check if parse args already called. If so, sync task parameters with parser
                if argparser_parseargs_called():
                    for parser, parsed_args in get_argparser_last_args():
                        task._connect_argparse(parser=parser, parsed_args=parsed_args)
            elif argparser_parseargs_called():
                # actually we have nothing to do, in remote running, the argparser will ignore
                # all non argparser parameters, only caveat if parameter connected with the same name
                # as the argparser this will be solved once sections are introduced to parameters
                pass

        # Make sure we start the logger, it will patch the main logging object and pipe all output
        # if we are running locally and using development mode worker, we will pipe all stdout to logger.
        # The logger will automatically take care of all patching (we just need to make sure to initialize it)
        logger = task._get_logger(auto_connect_streams=auto_connect_streams)
        # show the debug metrics page in the log, it is very convenient
        if not is_sub_process_task_id:
            if cls._offline_mode:
                logger.report_text('ClearML running in offline mode, session stored in {}'.format(
                    task.get_offline_mode_folder()))
            else:
                logger.report_text('ClearML results page: {}'.format(task.get_output_log_web_page()))
        # Make sure we start the dev worker if required, otherwise it will only be started when we write
        # something to the log.
        task._dev_mode_setup_worker()

        if (not task._reporter or not task._reporter.is_alive()) and \
                is_sub_process_task_id and not cls._report_subprocess_enabled:
            task._setup_reporter()

        # start monitoring in background process or background threads
        # monitoring are: Resource monitoring and Dev Worker monitoring classes
        BackgroundMonitor.start_all(task=task)

        return task

    @classmethod
    def create(
            cls,
            project_name=None,  # type: Optional[str]
            task_name=None,  # type: Optional[str]
            task_type=None,  # type: Optional[str]
            repo=None,  # type: Optional[str]
            branch=None,  # type: Optional[str]
            commit=None,  # type: Optional[str]
            script=None,  # type: Optional[str]
            working_directory=None,  # type: Optional[str]
            packages=None,  # type: Optional[Union[bool, Sequence[str]]]
            requirements_file=None,  # type: Optional[Union[str, Path]]
            docker=None,  # type: Optional[str]
            docker_args=None,  # type: Optional[str]
            docker_bash_setup_script=None,  # type: Optional[str]
            argparse_args=None,  # type: Optional[Sequence[Tuple[str, str]]]
            base_task_id=None,  # type: Optional[str]
            add_task_init_call=True,  # type: bool
    ):
        # type: (...) -> Task
        """
        Manually create and populate a new Task (experiment) in the system.
        If the code does not already contain a call to ``Task.init``, pass add_task_init_call=True,
        and the code will be patched in remote execution (i.e. when executed by `clearml-agent`

        .. note::
           This method **always** creates a new Task.
           Use :meth:`Task.init` method to automatically create and populate task for the running process.
           To reference an existing Task, call the  :meth:`Task.get_task` method .

        :param project_name: Set the project name for the task. Required if base_task_id is None.
        :param task_name: Set the name of the remote task. Required if base_task_id is None.
        :param task_type: Optional, The task type to be created. Supported values: 'training', 'testing', 'inference',
            'data_processing', 'application', 'monitor', 'controller', 'optimizer', 'service', 'qc', 'custom'
        :param repo: Remote URL for the repository to use, or path to local copy of the git repository
            Example: 'https://github.com/allegroai/clearml.git' or '~/project/repo'
        :param branch: Select specific repository branch/tag (implies the latest commit from the branch)
        :param commit: Select specific commit id to use (default: latest commit,
            or when used with local repository matching the local commit id)
        :param script: Specify the entry point script for the remote execution. When used in tandem with
            remote git repository the script should be a relative path inside the repository,
            for example: './source/train.py' . When used with local repository path it supports a
            direct path to a file inside the local repository itself, for example: '~/project/source/train.py'
        :param working_directory: Working directory to launch the script from. Default: repository root folder.
            Relative to repo root or local folder.
        :param packages: Manually specify a list of required packages. Example: ["tqdm>=2.1", "scikit-learn"]
            or `True` to automatically create requirements
            based on locally installed packages (repository must be local).
        :param requirements_file: Specify requirements.txt file to install when setting the session.
            If not provided, the requirements.txt from the repository will be used.
        :param docker: Select the docker image to be executed in by the remote session
        :param docker_args: Add docker arguments, pass a single string
        :param docker_bash_setup_script: Add bash script to be executed
            inside the docker before setting up the Task's environment
        :param argparse_args: Arguments to pass to the remote execution, list of string pairs (argument, value)
            Notice, only supported if the codebase itself uses argparse.ArgumentParser
        :param base_task_id: Use a pre-existing task in the system, instead of a local repo/script.
            Essentially clones an existing task and overrides arguments/requirements.
        :param add_task_init_call: If True, a 'Task.init()' call is added to the script entry point in remote execution.

        :return: The newly created Task (experiment)
        """
        if not project_name and not base_task_id:
            if not cls.__main_task:
                raise ValueError("Please provide project_name, no global task context found "
                                 "(Task.current_task hasn't been called)")
            project_name = cls.__main_task.get_project_name()
        from .backend_interface.task.populate import CreateAndPopulate
        manual_populate = CreateAndPopulate(
            project_name=project_name, task_name=task_name, task_type=task_type,
            repo=repo, branch=branch, commit=commit,
            script=script, working_directory=working_directory,
            packages=packages, requirements_file=requirements_file,
            docker=docker, docker_args=docker_args, docker_bash_setup_script=docker_bash_setup_script,
            base_task_id=base_task_id,
            add_task_init_call=add_task_init_call,
            raise_on_missing_entries=False,
        )
        task = manual_populate.create_task()
        if task and argparse_args:
            manual_populate.update_task_args(argparse_args)
            task.reload()

        return task

    @classmethod
    def get_task(
            cls,
            task_id=None,  # type: Optional[str]
            project_name=None,  # type: Optional[str]
            task_name=None,  # type: Optional[str]
            tags=None,  # type: Optional[Sequence[str]]
            allow_archived=True,  # type: bool
            task_filter=None  # type: Optional[dict]
    ):
        # type: (...) -> "Task"
        """
        Get a Task by Id, or project name / task name combination.

        For example:

        The following code demonstrates calling ``Task.get_task`` to report a scalar to another Task. The output
        of :meth:`.Logger.report_scalar` from testing is associated with the Task named ``training``. It allows
        training and testing to run concurrently, because they initialized different Tasks (see :meth:`Task.init`
        for information about initializing Tasks).

        The training script:

        .. code-block:: py

            # initialize the training Task
            task = Task.init('myProject', 'training')

            # do some training

        The testing script:

        .. code-block:: py

            # initialize the testing Task
            task = Task.init('myProject', 'testing')

            # get the training Task
            train_task = Task.get_task(project_name='myProject', task_name='training')

            # report metrics in the training Task
            for x in range(10):
                train_task.get_logger().report_scalar('title', 'series', value=x * 2, iteration=x)

        :param str task_id: The Id (system UUID) of the experiment to get.
            If specified, ``project_name`` and ``task_name`` are ignored.
        :param str project_name: The project name of the Task to get.
        :param str task_name: The name of the Task within ``project_name`` to get.
        :param list tags: Filter based on the requested list of tags (strings) (Task must have all the listed tags)
            To exclude a tag add "-" prefix to the tag. Example: ["best", "-debug"]
        :param bool allow_archived: Only applicable if *not* using specific ``task_id``,
            If True (default) allow to return archived Tasks, if False filter out archived Tasks
        :param bool task_filter: Only applicable if *not* using specific ``task_id``,
            Pass additional query filters, on top of project/name. See details in Task.get_tasks.

        :return: The Task specified by ID, or project name / experiment name combination.
        """
        return cls.__get_task(
            task_id=task_id, project_name=project_name, task_name=task_name, tags=tags,
            include_archived=allow_archived, task_filter=task_filter,
        )

    @classmethod
    def get_tasks(
            cls,
            task_ids=None,  # type: Optional[Sequence[str]]
            project_name=None,  # type: Optional[Union[Sequence[str],str]]
            task_name=None,  # type: Optional[str]
            tags=None,  # type: Optional[Sequence[str]]
            task_filter=None  # type: Optional[Dict]
    ):
        # type: (...) -> List["Task"]
        """
        Get a list of Tasks objects matching the queries/filters

        - A list of specific Task IDs.
        - Filter Tasks based on specific fields:
            project name (including partial match), task name (including partial match), tags
            Apply Additional advanced filtering with `task_filter`

        :param list(str) task_ids: The Ids (system UUID) of experiments to get.
            If ``task_ids`` specified, then ``project_name`` and ``task_name`` are ignored.
        :param str project_name: The project name of the Tasks to get. To get the experiment
            in all projects, use the default value of ``None``. (Optional)
            Use a list of string for multiple optional project names.
        :param str task_name: The full name or partial name of the Tasks to match within the specified
            ``project_name`` (or all projects if ``project_name`` is ``None``).
            This method supports regular expressions for name matching. (Optional)
        :param list(str) task_ids: list of unique task id string (if exists other parameters are ignored)
        :param str project_name: project name (str) the task belongs to (use None for all projects)
        :param str task_name: task name (str) in within the selected project
            Return any partial match of task_name, regular expressions matching is also supported
            If None is passed, returns all tasks within the project
        :param list tags: Filter based on the requested list of tags (strings) (Task must have all the listed tags)
            To exclude a tag add "-" prefix to the tag. Example: ["best", "-debug"]
        :param dict task_filter: filter and order Tasks. See service.tasks.GetAllRequest for details
            `parent`: (str) filter by parent task-id matching
            `search_text`: (str) free text search (in task fields comment/name/id)
            `status`: List[str] List of valid statuses
                (options are: "created", "queued", "in_progress", "stopped", "published", "completed")
            `type`: List[str] List of valid task type
                (options are: 'training', 'testing', 'inference', 'data_processing', 'application', 'monitor',
                'controller', 'optimizer', 'service', 'qc'. 'custom')
            `user`: List[str] Filter based on Task's user owner, provide list of valid user Ids.
            `order_by`: List[str] List of field names to order by. When search_text is used,
                Use '-' prefix to specify descending order. Optional, recommended when using page
                Example: order_by=['-last_update']
            `_all_`: dict(fields=[], pattern='')  Match string `pattern` (regular expression)
                appearing in All `fields`
                dict(fields=['script.repository'], pattern='github.com/user')
            `_any_`: dict(fields=[], pattern='')  Match string `pattern` (regular expression)
                appearing in Any of the `fields`
                dict(fields=['comment', 'name'], pattern='my comment')
            Examples:
                {'status': ['stopped'], 'order_by': ["-last_update"]}
                {'order_by'=['-last_update'], '_all_'=dict(fields=['script.repository'], pattern='github.com/user'))

        :return: The Tasks specified by the parameter combinations (see the parameters).
        """
        return cls.__get_tasks(task_ids=task_ids, project_name=project_name, tags=tags,
                               task_name=task_name, **(task_filter or {}))

    @classmethod
    def query_tasks(
            cls,
            project_name=None,  # type: Optional[Union[Sequence[str],str]]
            task_name=None,  # type: Optional[str]
            tags=None,  # type: Optional[Sequence[str]]
            additional_return_fields=None,  # type: Optional[Sequence[str]]
            task_filter=None,  # type: Optional[Dict]
    ):
        # type: (...) -> Union[List[str], List[Dict[str, str]]]
        """
        Get a list of Tasks ID matching the specific query/filter.
        Notice, if `additional_return_fields` is specified, returns a list of
        dictionaries with requested fields (dict per Task)

        :param str project_name: The project name of the Tasks to get. To get the experiment
            in all projects, use the default value of ``None``. (Optional)
            Use a list of string for multiple optional project names.
        :param str task_name: The full name or partial name of the Tasks to match within the specified
            ``project_name`` (or all projects if ``project_name`` is ``None``).
            This method supports regular expressions for name matching. (Optional)
        :param str project_name: project name (str) the task belongs to (use None for all projects)
        :param str task_name: task name (str) in within the selected project
            Return any partial match of task_name, regular expressions matching is also supported
            If None is passed, returns all tasks within the project
        :param list tags: Filter based on the requested list of tags (strings) (Task must have all the listed tags)
            To exclude a tag add "-" prefix to the tag. Example: ["best", "-debug"]
        :param list additional_return_fields: Optional, if not provided return a list of Task IDs.
            If provided return dict per Task with the additional requested fields.
            Example: returned_fields=['last_updated', 'user', 'script.repository'] will return a list of dict:
                [{'id': 'task_id', 'last_update': datetime.datetime(),
                  'user': 'user_id', 'script.repository': 'https://github.com/user/'}, ]
        :param dict task_filter: filter and order Tasks. See service.tasks.GetAllRequest for details
            `parent`: (str) filter by parent task-id matching
            `search_text`: (str) free text search (in task fields comment/name/id)
            `status`: List[str] List of valid statuses
                (options are: "created", "queued", "in_progress", "stopped", "published", "completed")
            `type`: List[str] List of valid task type
                (options are: 'training', 'testing', 'inference', 'data_processing', 'application', 'monitor',
                'controller', 'optimizer', 'service', 'qc'. 'custom')
            `user`: List[str] Filter based on Task's user owner, provide list of valid user Ids.
            `order_by`: List[str] List of field names to order by. When search_text is used,
                Use '-' prefix to specify descending order. Optional, recommended when using page
                Example: order_by=['-last_update']
            `_all_`: dict(fields=[], pattern='')  Match string `pattern` (regular expression)
                appearing in All `fields`
                dict(fields=['script.repository'], pattern='github.com/user')
            `_any_`: dict(fields=[], pattern='')  Match string `pattern` (regular expression)
                appearing in Any of the `fields`
                dict(fields=['comment', 'name'], pattern='my comment')
            Examples:
                {'status': ['stopped'], 'order_by': ["-last_update"]}
                {'order_by'=['-last_update'], '_all_'=dict(fields=['script.repository'], pattern='github.com/user'))
        :return: The Tasks specified by the parameter combinations (see the parameters).
        """
        if tags:
            task_filter = task_filter or {}
            task_filter['tags'] = (task_filter.get('tags') or []) + list(tags)
        return_fields = {}
        if additional_return_fields:
            task_filter = task_filter or {}
            return_fields = set(list(additional_return_fields) + ['id'])
            task_filter['only_fields'] = (task_filter.get('only_fields') or []) + list(return_fields)

        results = cls._query_tasks(project_name=project_name, task_name=task_name, **(task_filter or {}))
        return [t.id for t in results] if not additional_return_fields else \
            [{k: cls._get_data_property(prop_path=k, data=r, raise_on_error=False, log_on_error=False)
              for k in return_fields}
             for r in results]

    @property
    def output_uri(self):
        # type: () -> str
        return self.storage_uri

    @output_uri.setter
    def output_uri(self, value):
        # type: (Union[str, bool]) -> None

        # check if this is boolean
        if value is False:
            value = None
        elif value is True:
            value = self.__default_output_uri or self._get_default_report_storage_uri()

        # check if we have the correct packages / configuration
        if value and value != self.storage_uri:
            from .storage.helper import StorageHelper
            helper = StorageHelper.get(value)
            if not helper:
                raise ValueError("Could not get access credentials for '{}' "
                                 ", check configuration file ~/clearml.conf".format(value))
            helper.check_write_permissions(value)
        self.storage_uri = value

    @property
    def artifacts(self):
        # type: () -> Dict[str, Artifact]
        """
        A read-only dictionary of Task artifacts (name, artifact).

        :return: The artifacts.
        """
        if not Session.check_min_api_version('2.3'):
            return ReadOnlyDict()
        artifacts_pairs = []
        if self.data.execution and self.data.execution.artifacts:
            artifacts_pairs = [(a.key, Artifact(a)) for a in self.data.execution.artifacts]
        if self._artifacts_manager:
            artifacts_pairs += list(self._artifacts_manager.registered_artifacts.items())
        return ReadOnlyDict(artifacts_pairs)

    @property
    def models(self):
        # type: () -> Mapping[str, Sequence[Model]]
        """
        Read-only dictionary of the Task's loaded/stored models.

        :return: A dictionary-like object with "input"/"output" keys and input/output properties, pointing to a
            list-like object containing of Model objects. Each list-like object also acts as a dictionary, mapping
            model name to a appropriate model instance.

            Get input/output models:

            .. code-block:: py

                task.models.input
                task.models["input"]

                task.models.output
                task.models["output"]

            Get the last output model:

            .. code-block:: py

                task.models.output[-1]

            Get a model by name:

            .. code-block:: py

                task.models.output["model name"]
        """
        return self.get_models()

    @property
    def logger(self):
        # type: () -> Logger
        """
        Get a Logger object for reporting, for this task context. You can view all Logger report output associated with
        the Task for which this method is called, including metrics, plots, text, tables, and images, in the
        **ClearML Web-App (UI)**.

        :return: The Logger object for the current Task (experiment).
        """
        return self.get_logger()

    @classmethod
    def clone(
            cls,
            source_task=None,  # type: Optional[Union[Task, str]]
            name=None,  # type: Optional[str]
            comment=None,  # type: Optional[str]
            parent=None,  # type: Optional[str]
            project=None,  # type: Optional[str]
    ):
        # type: (...) -> Task
        """
        Create a duplicate (a clone) of a Task (experiment). The status of the cloned Task is ``Draft``
        and modifiable.

        Use this method to manage experiments and for autoML.

        :param str source_task: The Task to clone. Specify a Task object or a  Task ID. (Optional)
        :param str name: The name of the new cloned Task. (Optional)
        :param str comment: A comment / description for the new cloned Task. (Optional)
        :param str parent: The Id of the parent Task of the new Task.

            - If ``parent`` is not specified, then ``parent`` is set to ``source_task.parent``.
            - If ``parent`` is not specified and ``source_task.parent`` is not available, then
              ``parent`` set to to ``source_task``.

        :param str project: The Id of the project in which to create the new Task.
            If ``None``, the new task inherits the original Task's project. (Optional)

        :return: The new cloned Task (experiment).
        """
        assert isinstance(source_task, (six.string_types, Task))
        if not Session.check_min_api_version('2.4'):
            raise ValueError("ClearML-server does not support DevOps features, "
                             "upgrade clearml-server to 0.12.0 or above")

        task_id = source_task if isinstance(source_task, six.string_types) else source_task.id
        if not parent:
            if isinstance(source_task, six.string_types):
                source_task = cls.get_task(task_id=source_task)
            parent = source_task.id if not source_task.parent else source_task.parent
        elif isinstance(parent, Task):
            parent = parent.id

        cloned_task_id = cls._clone_task(cloned_task_id=task_id, name=name, comment=comment,
                                         parent=parent, project=project)
        cloned_task = cls.get_task(task_id=cloned_task_id)
        return cloned_task

    @classmethod
    def enqueue(cls, task, queue_name=None, queue_id=None):
        # type: (Union[Task, str], Optional[str], Optional[str]) -> Any
        """
        Enqueue a Task for execution, by adding it to an execution queue.

        .. note::
           A worker daemon must be listening at the queue for the worker to fetch the Task and execute it,
           see `Use Case Examples <../clearml_agent_ref/#use-case-examples>`_ on the "ClearML Agent
           Reference page.

        :param Task/str task: The Task to enqueue. Specify a Task object or  Task ID.
        :param str queue_name: The name of the queue. If not specified, then ``queue_id`` must be specified.
        :param str queue_id: The Id of the queue. If not specified, then ``queue_name`` must be specified.

        :return: An enqueue JSON response.

            .. code-block:: javascript

               {
                    "queued": 1,
                    "updated": 1,
                    "fields": {
                        "status": "queued",
                        "status_reason": "",
                        "status_message": "",
                        "status_changed": "2020-02-24T15:05:35.426770+00:00",
                        "last_update": "2020-02-24T15:05:35.426770+00:00",
                        "execution.queue": "2bd96ab2d9e54b578cc2fb195e52c7cf"
                        }
                }

            - ``queued``  - The number of Tasks enqueued (an integer or ``null``).
            - ``updated`` - The number of Tasks updated (an integer or ``null``).
            - ``fields``

              - ``status`` - The status of the experiment.
              - ``status_reason`` - The reason for the last status change.
              - ``status_message`` - Information about the status.
              - ``status_changed`` - The last status change date and time (ISO 8601 format).
              - ``last_update`` - The last Task update time, including Task creation, update, change, or events for
                this task (ISO 8601 format).
              - ``execution.queue`` - The Id of the queue where the Task is enqueued. ``null`` indicates not enqueued.

        """
        assert isinstance(task, (six.string_types, Task))
        if not Session.check_min_api_version('2.4'):
            raise ValueError("ClearML-server does not support DevOps features, "
                             "upgrade clearml-server to 0.12.0 or above")

        # make sure we have wither name ot id
        mutually_exclusive(queue_name=queue_name, queue_id=queue_id)

        task_id = task if isinstance(task, six.string_types) else task.id
        session = cls._get_default_session()
        if not queue_id:
            queue_id = get_queue_id(session, queue_name)
            if not queue_id:
                raise ValueError('Could not find queue named "{}"'.format(queue_name))

        req = tasks.EnqueueRequest(task=task_id, queue=queue_id)
        res = cls._send(session=session, req=req)
        if not res.ok():
            raise ValueError(res.response)
        resp = res.response
        return resp

    @classmethod
    def dequeue(cls, task):
        # type: (Union[Task, str]) -> Any
        """
        Dequeue (remove) a Task from an execution queue.

        :param Task/str task: The Task to dequeue. Specify a Task object or  Task ID.

        :return: A dequeue JSON response.

        .. code-block:: javascript

           {
                "dequeued": 1,
                "updated": 1,
                "fields": {
                    "status": "created",
                    "status_reason": "",
                    "status_message": "",
                    "status_changed": "2020-02-24T16:43:43.057320+00:00",
                    "last_update": "2020-02-24T16:43:43.057320+00:00",
                    "execution.queue": null
                    }
            }

        - ``dequeued``  - The number of Tasks enqueued (an integer or ``null``).
        - ``fields``

          - ``status`` - The status of the experiment.
          - ``status_reason`` - The reason for the last status change.
          - ``status_message`` - Information about the status.
          - ``status_changed`` - The last status change date and time in ISO 8601 format.
          - ``last_update`` - The last time the Task was created, updated,
                changed or events for this task were reported.
          - ``execution.queue`` - The Id of the queue where the Task is enqueued. ``null`` indicates not enqueued.

        - ``updated`` - The number of Tasks updated (an integer or ``null``).

        """
        assert isinstance(task, (six.string_types, Task))
        if not Session.check_min_api_version('2.4'):
            raise ValueError("ClearML-server does not support DevOps features, "
                             "upgrade clearml-server to 0.12.0 or above")

        task_id = task if isinstance(task, six.string_types) else task.id
        session = cls._get_default_session()
        req = tasks.DequeueRequest(task=task_id)
        res = cls._send(session=session, req=req)
        resp = res.response
        return resp

    def add_tags(self, tags):
        # type: (Union[Sequence[str], str]) -> None
        """
        Add Tags to this task. Old tags are not deleted. When executing a Task (experiment) remotely,
        this method has no effect).

        :param tags: A list of tags which describe the Task to add.
        """

        if isinstance(tags, six.string_types):
            tags = tags.split(" ")

        self.data.tags = list(set((self.data.tags or []) + tags))
        self._edit(tags=self.data.tags)

    def connect(self, mutable, name=None):
        # type: (Any, Optional[str]) -> Any
        """
        Connect an object to a Task object. This connects an experiment component (part of an experiment) to the
        experiment. For example, connect hyperparameters or models.

        :param object mutable: The experiment component to connect. The object can be any object Task supports
            integrating, including:

            - argparse - An argparse object for parameters.
            - dict - A dictionary for parameters.
            - TaskParameters - A TaskParameters object.
            - Model - A model object for initial model warmup, or for model update/snapshot uploading.
            - Class type - A Class type, storing all class properties (excluding '_' prefix properties)
            - Object - A class instance, storing all instance properties (excluding '_' prefix properties)

        :param str name: A section name associated with the connected object, if 'name' is None defaults to 'General'
            Currently only supported for `dict` / `TaskParameter` objects
            Examples:
            name='General' will put the connected dictionary under the General section in the hyper-parameters
            name='Train' will put the connected dictionary under the Train section in the hyper-parameters

        :return: The result returned when connecting the object, if supported.

        :raise: Raise an exception on unsupported objects.
        """
        # dispatching by match order
        dispatch = (
            (OutputModel, self._connect_output_model),
            (InputModel, self._connect_input_model),
            (ArgumentParser, self._connect_argparse),
            (dict, self._connect_dictionary),
            (TaskParameters, self._connect_task_parameters),
            (type, self._connect_object),
            (object, self._connect_object),
        )

        multi_config_support = Session.check_min_api_version('2.9')
        if multi_config_support and not name and not isinstance(mutable, (OutputModel, InputModel)):
            name = self._default_configuration_section_name

        if not multi_config_support and name and name != self._default_configuration_section_name:
            raise ValueError("Multiple configurations is not supported with the current 'clearml-server', "
                             "please upgrade to the latest version")

        for mutable_type, method in dispatch:
            if isinstance(mutable, mutable_type):
                return method(mutable, name=name)

        raise Exception('Unsupported mutable type %s: no connect function found' % type(mutable).__name__)

    def connect_configuration(self, configuration, name=None, description=None):
        # type: (Union[Mapping, list, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str]
        """
        Connect a configuration dictionary or configuration file (pathlib.Path / str) to a Task object.
        This method should be called before reading the configuration file.

        Later, when creating an output model, the model will include the contents of the configuration dictionary
        or file.

        For example, a local file:

        .. code-block:: py

           config_file = task.connect_configuration(config_file)
           my_params = json.load(open(config_file,'rt'))

        A parameter dictionary/list:

        .. code-block:: py

           my_params = task.connect_configuration(my_params)

        :param configuration: The configuration. This is usually the configuration used in the model training process.
            Specify one of the following:

            - A dictionary/list - A dictionary containing the configuration. ClearML stores the configuration in
              the **ClearML Server** (backend), in a HOCON format (JSON-like format) which is editable.
            - A ``pathlib2.Path`` string - A path to the configuration file. ClearML stores the content of the file.
              A local path must be relative path. When executing a Task remotely in a worker, the contents brought
              from the **ClearML Server** (backend) overwrites the contents of the file.

        :param str name: Configuration section name. default: 'General'
            Allowing users to store multiple configuration dicts/files

        :param str description: Configuration section description (text). default: None

        :return: If a dictionary is specified, then a dictionary is returned. If pathlib2.Path / string is
            specified, then a path to a local configuration file is returned. Configuration object.
        """
        pathlib_Path = None  # noqa
        if not isinstance(configuration, (dict, list, Path, six.string_types)):
            try:
                from pathlib import Path as pathlib_Path  # noqa
            except ImportError:
                pass
            if not pathlib_Path or not isinstance(configuration, pathlib_Path):
                raise ValueError("connect_configuration supports `dict`, `str` and 'Path' types, "
                                 "{} is not supported".format(type(configuration)))

        multi_config_support = Session.check_min_api_version('2.9')
        if multi_config_support and not name:
            name = self._default_configuration_section_name

        if not multi_config_support and name and name != self._default_configuration_section_name:
            raise ValueError("Multiple configurations is not supported with the current 'clearml-server', "
                             "please upgrade to the latest version")

        # parameter dictionary
        if isinstance(configuration, (dict, list,)):
            def _update_config_dict(task, config_dict):
                if multi_config_support:
                    # noinspection PyProtectedMember
                    task._set_configuration(
                        name=name, description=description, config_type='dictionary', config_dict=config_dict)
                else:
                    # noinspection PyProtectedMember
                    task._set_model_config(config_dict=config_dict)

            if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()):
                if multi_config_support:
                    self._set_configuration(
                        name=name, description=description, config_type='dictionary', config_dict=configuration)
                else:
                    self._set_model_config(config_dict=configuration)
                if isinstance(configuration, dict):
                    configuration = ProxyDictPostWrite(self, _update_config_dict, **configuration)
            else:
                # noinspection PyBroadException
                try:
                    remote_configuration = self._get_configuration_dict(name=name) \
                        if multi_config_support else self._get_model_config_dict()
                except Exception:
                    remote_configuration = None

                if remote_configuration is None:
                    LoggerRoot.get_base_logger().warning(
                        "Could not retrieve remote configuration named \'{}\'\n"
                        "Using default configuration: {}".format(name, str(configuration)))
                    # update back configuration section
                    if multi_config_support:
                        self._set_configuration(
                            name=name, description=description,
                            config_type='dictionary', config_dict=configuration)
                    return configuration

                if isinstance(configuration, dict):
                    configuration.clear()
                    configuration.update(remote_configuration)
                    configuration = ProxyDictPreWrite(False, False, **configuration)
                elif isinstance(configuration, list):
                    configuration.clear()
                    configuration.extend(remote_configuration)

            return configuration

        # it is a path to a local file
        if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()):
            # check if not absolute path
            configuration_path = Path(configuration)
            if not configuration_path.is_file():
                ValueError("Configuration file does not exist")
            try:
                with open(configuration_path.as_posix(), 'rt') as f:
                    configuration_text = f.read()
            except Exception:
                raise ValueError("Could not connect configuration file {}, file could not be read".format(
                    configuration_path.as_posix()))
            if multi_config_support:
                self._set_configuration(
                    name=name, description=description,
                    config_type=configuration_path.suffixes[-1].lstrip('.')
                    if configuration_path.suffixes and configuration_path.suffixes[-1] else 'file',
                    config_text=configuration_text)
            else:
                self._set_model_config(config_text=configuration_text)
            return configuration
        else:
            configuration_text = self._get_configuration_text(name=name) if multi_config_support \
                else self._get_model_config_text()
            if configuration_text is None:
                LoggerRoot.get_base_logger().warning(
                    "Could not retrieve remote configuration named \'{}\'\n"
                    "Using default configuration: {}".format(name, str(configuration)))
                # update back configuration section
                if multi_config_support:
                    configuration_path = Path(configuration)
                    if configuration_path.is_file():
                        with open(configuration_path.as_posix(), 'rt') as f:
                            configuration_text = f.read()

                        self._set_configuration(
                            name=name, description=description,
                            config_type=configuration_path.suffixes[-1].lstrip('.')
                            if configuration_path.suffixes and configuration_path.suffixes[-1] else 'file',
                            config_text=configuration_text)
                return configuration

            configuration_path = Path(configuration)
            fd, local_filename = mkstemp(prefix='clearml_task_config_',
                                         suffix=configuration_path.suffixes[-1] if
                                         configuration_path.suffixes else '.txt')
            os.write(fd, configuration_text.encode('utf-8'))
            os.close(fd)
            if pathlib_Path:
                return pathlib_Path(local_filename)
            return Path(local_filename) if isinstance(configuration, Path) else local_filename

    def connect_label_enumeration(self, enumeration):
        # type: (Dict[str, int]) -> Dict[str, int]
        """
        Connect a label enumeration dictionary to a Task (experiment) object.

        Later, when creating an output model, the model will include the label enumeration dictionary.

        :param dict enumeration: A label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        :return: The label enumeration dictionary (JSON).
        """
        if not isinstance(enumeration, dict):
            raise ValueError("connect_label_enumeration supports only `dict` type, "
                             "{} is not supported".format(type(enumeration)))

        if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()):
            self.set_model_label_enumeration(enumeration)
        else:
            # pop everything
            enumeration.clear()
            enumeration.update(self.get_labels_enumeration())
        return enumeration

    def get_logger(self):
        # type: () -> Logger
        """
        Get a Logger object for reporting, for this task context. You can view all Logger report output associated with
        the Task for which this method is called, including metrics, plots, text, tables, and images, in the
        **ClearML Web-App (UI)**.

        :return: The Logger for the Task (experiment).
        """
        return self._get_logger(auto_connect_streams=self._log_to_backend)

    def mark_started(self, force=False):
        # type: (bool) -> ()
        """
        Manually mark a Task as started (happens automatically)

        :param bool force: If True the task status will be changed to `started` regardless of the current Task state.
        """
        # UI won't let us see metrics if we're not started
        self.started(force=force)
        self.reload()

    def mark_stopped(self, force=False, status_message=None):
        # type: (bool, Optional[str]) -> ()
        """
        Manually mark a Task as stopped (also used in :meth:`_at_exit`)

        :param bool force: If True the task status will be changed to `stopped` regardless of the current Task state.
        :param str status_message: Optional, add status change message to the stop request.
            This message will be stored as status_message on the Task's info panel
        """
        # flush any outstanding logs
        self.flush(wait_for_uploads=True)
        # mark task as stopped
        self.stopped(force=force, status_message=str(status_message) if status_message else None)

    def flush(self, wait_for_uploads=False):
        # type: (bool) -> bool
        """
        Flush any outstanding reports or console logs.

        :param bool wait_for_uploads: Wait for all outstanding uploads to complete

            - ``True`` - Wait
            - ``False`` - Do not wait (default)
        """

        # make sure model upload is done
        if BackendModel.get_num_results() > 0 and wait_for_uploads:
            BackendModel.wait_for_results()

        # flush any outstanding logs
        if self._logger:
            # noinspection PyProtectedMember
            self._logger._flush_stdout_handler()
        if self.__reporter:
            self.__reporter.flush()
            if wait_for_uploads:
                self.__reporter.wait_for_events()

        LoggerRoot.flush()

        return True

    def reset(self, set_started_on_success=False, force=False):
        # type: (bool, bool) -> None
        """
        Reset a Task. ClearML reloads a Task after a successful reset.
        When a worker executes a Task remotely, the Task does not reset unless
        the ``force`` parameter is set to ``True`` (this avoids accidentally clearing logs and metrics).

        :param bool set_started_on_success: If successful, automatically set the Task to `started`

            - ``True`` - If successful, set to started.
            - ``False`` - If successful, do not set to started. (default)

        :param bool force: Force a Task reset, even when executing the Task (experiment) remotely in a worker

            - ``True`` - Force
            - ``False`` - Do not force (default)
        """
        if not running_remotely() or not self.is_main_task() or force:
            super(Task, self).reset(set_started_on_success=set_started_on_success, force=force)

    def close(self):
        """
        Close the current Task. Enables you to manually shutdown the task.

        .. warning::
           Only call :meth:`Task.close` if you are certain the Task is not needed.
        """
        if self._at_exit_called:
            return

        # store is main before we call at_exit, because will will Null it
        is_main = self.is_main_task()
        is_sub_process = self.__is_subprocess()

        # wait for repository detection (5 minutes should be reasonable time to detect all packages)
        if self._logger and not self.__is_subprocess():
            self._wait_for_repo_detection(timeout=300.)

        self.__shutdown()
        # unregister atexit callbacks and signal hooks, if we are the main task
        if is_main:
            self.__register_at_exit(None)
            if not is_sub_process:
                # make sure we enable multiple Task.init callas with reporting sub-processes
                BackgroundMonitor.clear_main_process(self)
                # noinspection PyProtectedMember
                Logger._remove_std_logger()

    def delete(self, delete_artifacts_and_models=True, skip_models_used_by_other_tasks=True, raise_on_error=False):
        # type: (bool, bool, bool) -> bool
        """
        Delete the task as well as it's output models and artifacts.
        Models and artifacts are deleted from their storage locations, each using its URI.

        Note: in order to delete models and artifacts using their URI, make sure the proper storage credentials are
        configured in your configuration file (e.g. if an artifact is stored in S3, make sure sdk.aws.s3.credentials
        are properly configured and that you have delete permission in the related buckets).

        :param delete_artifacts_and_models: If True, artifacts and models would also be deleted (default True)
        :param skip_models_used_by_other_tasks: If True, models used by other tasks would not be deleted (default True)
        :param raise_on_error: If True an exception will be raised when encountering an error.
                               If False an error would be printed and no exception will be raised.
        :return: True if the task was deleted successfully.
        """
        if not running_remotely() or not self.is_main_task():
            return super(Task, self)._delete(
                delete_artifacts_and_models=delete_artifacts_and_models,
                skip_models_used_by_other_tasks=skip_models_used_by_other_tasks,
                raise_on_error=raise_on_error,
            )
        return False

    def register_artifact(self, name, artifact, metadata=None, uniqueness_columns=True):
        # type: (str, pandas.DataFrame, Dict, Union[bool, Sequence[str]]) -> None
        """
        Register (add) an artifact for the current Task. Registered artifacts are dynamically sychronized with the
        **ClearML Server** (backend). If a registered artifact is updated, the update is stored in the
        **ClearML Server** (backend). Registered artifacts are primarily used for Data Auditing.

        The currently supported registered artifact object type is a pandas.DataFrame.

        See also :meth:`Task.unregister_artifact` and :meth:`Task.get_registered_artifacts`.

        .. note::
           ClearML also supports uploaded artifacts which are one-time uploads of static artifacts that are not
           dynamically sychronized with the **ClearML Server** (backend). These static artifacts include
           additional object types. For more information, see :meth:`Task.upload_artifact`.

        :param str name: The name of the artifact.

         .. warning::
            If an artifact with the same name was previously registered, it is overwritten.
        :param object artifact: The artifact object.
        :param dict metadata: A dictionary of key-value pairs for any metadata. This dictionary appears with the
            experiment in the **ClearML Web-App (UI)**, **ARTIFACTS** tab.
        :param uniqueness_columns: A Sequence of columns for artifact uniqueness comparison criteria, or the default
            value of ``True``. If ``True``, the artifact uniqueness comparison criteria is all the columns,
            which is the same as ``artifact.columns``.
        """
        if not isinstance(uniqueness_columns, CollectionsSequence) and uniqueness_columns is not True:
            raise ValueError('uniqueness_columns should be a List (sequence) or True')
        if isinstance(uniqueness_columns, str):
            uniqueness_columns = [uniqueness_columns]
        self._artifacts_manager.register_artifact(
            name=name, artifact=artifact, metadata=metadata, uniqueness_columns=uniqueness_columns)

    def unregister_artifact(self, name):
        # type: (str) -> None
        """
        Unregister (remove) a registered artifact. This removes the artifact from the watch list that ClearML uses
        to synchronize artifacts with the **ClearML Server** (backend).

        .. important::
           - Calling this method does not remove the artifact from a Task. It only stops ClearML from
             monitoring the artifact.
           - When this method is called, ClearML immediately takes the last snapshot of the artifact.
        """
        self._artifacts_manager.unregister_artifact(name=name)

    def get_registered_artifacts(self):
        # type: () -> Dict[str, Artifact]
        """
        Get a dictionary containing the Task's registered (dynamically synchronized) artifacts (name, artifact object).

        .. note::
           After calling ``get_registered_artifacts``, you can still modify the registered artifacts.

        :return: The registered (dynamically synchronized) artifacts.
        """
        return self._artifacts_manager.registered_artifacts

    def upload_artifact(
            self,
            name,  # type: str
            artifact_object,  # type: Union[str, Mapping, pandas.DataFrame, numpy.ndarray, Image.Image, Any]
            metadata=None,  # type: Optional[Mapping]
            delete_after_upload=False,  # type: bool
            auto_pickle=True,  # type: bool
            preview=None,  # type: Any
            wait_on_upload=False,  # type: bool
    ):
        # type: (...) -> bool
        """
        Upload (add) a static artifact to a Task object. The artifact is uploaded in the background.

        The currently supported upload (static) artifact types include:

        - string / pathlib2.Path - A path to artifact file. If a wildcard or a folder is specified, then ClearML
          creates and uploads a ZIP file.
        - dict - ClearML stores a dictionary as ``.json`` file and uploads it.
        - pandas.DataFrame - ClearML stores a pandas.DataFrame as ``.csv.gz`` (compressed CSV) file and uploads it.
        - numpy.ndarray - ClearML stores a numpy.ndarray as ``.npz`` file and uploads it.
        - PIL.Image - ClearML stores a PIL.Image as ``.png`` file and uploads it.
        - Any - If called with auto_pickle=True, the object will be pickled and uploaded.

        :param str name: The artifact name.

            .. warning::
               If an artifact with the same name was previously uploaded, then it is overwritten.

        :param object artifact_object:  The artifact object.
        :param dict metadata: A dictionary of key-value pairs for any metadata. This dictionary appears with the
            experiment in the **ClearML Web-App (UI)**, **ARTIFACTS** tab.
        :param bool delete_after_upload: After the upload, delete the local copy of the artifact

            - ``True`` - Delete the local copy of the artifact.
            - ``False`` - Do not delete. (default)

        :param bool auto_pickle: If True (default) and the artifact_object is not one of the following types:
            pathlib2.Path, dict, pandas.DataFrame, numpy.ndarray, PIL.Image, url (string), local_file (string)
            the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl)

        :param Any preview: The artifact preview

        :param bool wait_on_upload: Whether or not the upload should be synchronous, forcing the upload to complete
            before continuing.

        :return: The status of the upload.

        - ``True`` - Upload succeeded.
        - ``False`` - Upload failed.

        :raise: If the artifact object type is not supported, raise a ``ValueError``.
        """
        return self._artifacts_manager.upload_artifact(
            name=name, artifact_object=artifact_object, metadata=metadata, delete_after_upload=delete_after_upload,
            auto_pickle=auto_pickle, preview=preview, wait_on_upload=wait_on_upload)

    def get_models(self):
        # type: () -> Mapping[str, Sequence[Model]]
        """
        Return a dictionary with {'input': [], 'output': []} loaded/stored models of the current Task
        Input models are files loaded in the task, either manually or automatically logged
        Output models are files stored in the task, either manually or automatically logged
        Automatically logged frameworks are for example: TensorFlow, Keras, PyTorch, ScikitLearn(joblib) etc.

        :return: A dictionary-like object with "input"/"output" keys and input/output properties, pointing to a
            list-like object containing of Model objects. Each list-like object also acts as a dictionary, mapping
            model name to a appropriate model instance.

            Example:

            .. code-block:: py

                {'input': [clearml.Model()], 'output': [clearml.Model()]}

        """
        return TaskModels(self)

    def is_current_task(self):
        # type: () -> bool
        """
        .. deprecated:: 0.13.0
           This method is deprecated. Use :meth:`Task.is_main_task` instead.

        Is this Task object the main execution Task (initially returned by :meth:`Task.init`)

        :return: Is this Task object the main execution Task

            - ``True`` - Is the main execution Task.
            - ``False`` - Is not the main execution Task.

        """
        return self.is_main_task()

    def is_main_task(self):
        # type: () -> bool
        """
        Is this Task object the main execution Task (initially returned by :meth:`Task.init`)

        .. note::
           If :meth:`Task.init` was never called, this method will *not* create
           it, making this test more efficient than:

           .. code-block:: py

              Task.init() == task

        :return: Is this Task object the main execution Task

            - ``True`` - Is the main execution Task.
            - ``False`` - Is not the main execution Task.

        """
        return self is self.__main_task

    def set_model_config(self, config_text=None, config_dict=None):
        # type: (Optional[str], Optional[Mapping]) -> None
        """
        .. deprecated:: 0.14.1
            Use :meth:`Task.connect_configuration` instead.
        """
        self._set_model_config(config_text=config_text, config_dict=config_dict)

    def get_model_config_text(self):
        # type: () -> str
        """
        .. deprecated:: 0.14.1
            Use :meth:`Task.connect_configuration` instead.
        """
        return self._get_model_config_text()

    def get_model_config_dict(self):
        # type: () -> Dict
        """
        .. deprecated:: 0.14.1
            Use :meth:`Task.connect_configuration` instead.
        """
        return self._get_model_config_dict()

    def set_model_label_enumeration(self, enumeration=None):
        # type: (Optional[Mapping[str, int]]) -> ()
        """
        Set the label enumeration for the Task object before creating an output model.
        Later, when creating an output model, the model will inherit these properties.

        :param dict enumeration: A label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }
        """
        super(Task, self).set_model_label_enumeration(enumeration=enumeration)

    def get_last_iteration(self):
        # type: () -> int
        """
        Get the last reported iteration, which is the last iteration for which the Task reported a metric.

        .. note::
           The maximum reported iteration is not in the local cache. This method
           sends a request to the **ClearML Server** (backend).

        :return: The last reported iteration number.
        """
        self._reload_last_iteration()
        return max(self.data.last_iteration or 0, self.__reporter.max_iteration if self.__reporter else 0)

    def set_initial_iteration(self, offset=0):
        # type: (int) -> int
        """
        Set initial iteration, instead of zero. Useful when continuing training from previous checkpoints

        :param int offset: Initial iteration (at starting point)
        :return: Newly set initial offset.
        """
        return super(Task, self).set_initial_iteration(offset=offset)

    def get_initial_iteration(self):
        # type: () -> int
        """
        Return the initial iteration offset, default is 0
        Useful when continuing training from previous checkpoints

        :return: Initial iteration offset.
        """
        return super(Task, self).get_initial_iteration()

    def get_last_scalar_metrics(self):
        # type: () -> Dict[str, Dict[str, Dict[str, float]]]
        """
        Get the last scalar metrics which the Task reported. This is a nested dictionary, ordered by title and series.

        For example:

        .. code-block:: javascript

           {
            'title': {
                'series': {
                    'last': 0.5,
                    'min': 0.1,
                    'max': 0.9
                    }
                }
            }

        :return: The last scalar metrics.
        """
        self.reload()
        metrics = self.data.last_metrics
        scalar_metrics = dict()
        for i in metrics.values():
            for j in i.values():
                scalar_metrics.setdefault(j['metric'], {}).setdefault(
                    j['variant'], {'last': j['value'], 'min': j['min_value'], 'max': j['max_value']})
        return scalar_metrics

    def get_parameters_as_dict(self):
        # type: () -> Dict
        """
        Get the Task parameters as a raw nested dictionary.

        .. note::
           The values are not parsed. They are returned as is.

        """
        return naive_nested_from_flat_dictionary(self.get_parameters())

    def set_parameters_as_dict(self, dictionary):
        # type: (Dict) -> None
        """
        Set the parameters for the Task object from a dictionary. The dictionary can be nested.
        This does not link the dictionary to the Task object. It does a one-time update. This
        is the same behavior as the :meth:`Task.connect` method.
        """
        self._arguments.copy_from_dict(flatten_dictionary(dictionary))

    def get_user_properties(self, value_only=False):
        # type: (bool) -> Dict[str, Union[str, dict]]
        """
        Get user properties for this task.
        Returns a dictionary mapping user property name to user property details dict.
        :param value_only: If True, returned user property details will be a string representing the property value.
        """
        if not Session.check_min_api_version("2.9"):
            self.log.info("User properties are not supported by the server")
            return {}

        section = "properties"

        params = self._hyper_params_manager.get_hyper_params(
            sections=[section], projector=attrgetter("value") if value_only else None
        )

        return dict(params.get(section, {}))

    def set_user_properties(
            self,
            *iterables,  # type: Union[Mapping[str, Union[str, dict, None]], Iterable[dict]]
            **properties  # type: Union[str, dict, int, float, None]
    ):
        # type: (...) -> bool
        """
        Set user properties for this task.
        A user property can contain the following fields (all of type string):
        name / value / description / type

        Examples:
            task.set_user_properties(backbone='great', stable=True)
            task.set_user_properties(backbone={"type": int, "description": "network type", "value": "great"}, )
            task.set_user_properties(
                {"name": "backbone", "description": "network type", "value": "great"},
                {"name": "stable", "description": "is stable", "value": True},
            )

        :param iterables: Properties iterables, each can be:
            * A dictionary of string key (name) to either a string value (value) a dict (property details). If the value
                is a dict, it must contain a "value" field. For example:

                .. code-block:: javascript

                    {
                        "property_name": {"description": "This is a user property", "value": "property value"},
                        "another_property_name": {"description": "This is user property", "value": "another value"},
                        "yet_another_property_name": "some value"
                    }

            * An iterable of dicts (each representing property details). Each dict must contain a "name" field and a
                "value" field. For example:

                .. code-block:: javascript

                    [
                        {
                            "name": "property_name",
                            "description": "This is a user property",
                            "value": "property value"
                        },
                        {
                            "name": "another_property_name",
                            "description": "This is another user property",
                            "value": "another value"
                        }
                    ]

        :param properties: Additional properties keyword arguments. Key is the property name, and value can be
            a string (property value) or a dict (property details). If the value is a dict, it must contain a "value"
            field. For example:

            .. code-block:: javascript

            {
                "property_name": "string as property value",
                "another_property_name": {
                    "type": "string",
                    "description": "This is user property",
                    "value": "another value"
                }
            }

        """
        if not Session.check_min_api_version("2.9"):
            self.log.info("User properties are not supported by the server")
            return False

        return self._hyper_params_manager.edit_hyper_params(
            iterables=list(properties.items()) + (
                list(iterables.items()) if isinstance(iterables, dict) else list(iterables)),
            replace='none',
            force_section="properties",
        )

    def get_script(self):
        # type: (...) -> Mapping[str, Optional[str]]
        """
        Get task's script details.

        Returns a dictionary containing the script details.
        :return: Dictionary with script properties e.g.
                {
                'working_dir': 'examples/reporting',
                'entry_point': 'artifacts.py',
                'branch': 'master',
                'repository': 'https://github.com/allegroai/clearml.git'
                }
        """
        script = self.data.script
        return {
            "working_dir": script.working_dir,
            "entry_point": script.entry_point,
            "branch": script.branch,
            "repository": script.repository
        }

    def set_script(
            self,
            repository=None,  # type: Optional[str]
            branch=None,  # type: Optional[str]
            commit=None,  # type: Optional[str]
            diff=None,  # type: Optional[str]
            working_dir=None,  # type: Optional[str]
            entry_point=None,  # type: Optional[str]
    ):
        # type: (...) -> None
        """
        Set task's script.

        Examples:
            task.set_script(repository='https://github.com/allegroai/clearml.git,
                            branch='main',
                            working_dir='examples/reporting',
                            entry_point='artifacts.py')

        :param repository: Optional, URL of remote repository. use empty string ("") to clear repository entry.
        :param branch: Optional, Select specific repository branch / tag. use empty string ("") to clear branch entry.
        :param commit: Optional, set specific git commit id. use empty string ("") to clear commit id entry.
        :param diff: Optional, set "git diff" section. use empty string ("") to clear git-diff entry.
        :param working_dir: Optional, Working directory to launch the script from.
        :param entry_point: Optional, Path to execute within the repository.

        """
        self.reload()
        script = self.data.script
        if repository is not None:
            script.repository = str(repository) or None
        if branch is not None:
            script.branch = str(branch) or None
            if script.tag:
                script.tag = None
        if commit is not None:
            script.version_num = str(commit) or None
        if diff is not None:
            script.diff = str(diff) or None
        if working_dir is not None:
            script.working_dir = str(working_dir)
        if entry_point is not None:
            script.entry_point = str(entry_point)
        # noinspection PyProtectedMember
        self._update_script(script=script)

    def delete_user_properties(self, *iterables):
        # type: (Iterable[Union[dict, Iterable[str, str]]]) -> bool
        """
        Delete hyper-parameters for this task.
        :param iterables: Hyper parameter key iterables. Each an iterable whose possible values each represent
        a hyper-parameter entry to delete, value formats are:

            * A dictionary containing a 'section' and 'name' fields
            * An iterable (e.g. tuple, list etc.) whose first two items denote 'section' and 'name'
        """
        if not Session.check_min_api_version("2.9"):
            self.log.info("User properties are not supported by the server")
            return False

        return self._hyper_params_manager.delete_hyper_params(*iterables)

    def set_base_docker(
            self,
            docker_cmd=None,  # type: Optional[str]
            docker_image=None,  # type: Optional[str]
            docker_arguments=None,  # type: Optional[Union[str, Sequence[str]]]
            docker_setup_bash_script=None  # type: Optional[Union[str, Sequence[str]]]
    ):
        # type: (...) -> ()
        """
        Set the base docker image for this experiment
        If provided, this value will be used by clearml-agent to execute this experiment
        inside the provided docker image.
        When running remotely the call is ignored

        :param docker_cmd: Deprecated! compound docker container image + arguments
            (example: 'nvidia/cuda:11.1 -e test=1') Deprecated, use specific arguments.
        :param docker_image: docker container image (example: 'nvidia/cuda:11.1')
        :param docker_arguments: docker execution parameters (example: '-e ENV=1')
        :param docker_setup_bash_script: bash script to run at the
            beginning of the docker before launching the Task itself. example: ['apt update', 'apt-get install -y gcc']
        """
        if not self.running_locally() and self.is_main_task():
            return

        super(Task, self).set_base_docker(
            docker_cmd=docker_cmd or docker_image,
            docker_arguments=docker_arguments,
            docker_setup_bash_script=docker_setup_bash_script
        )

    def set_resource_monitor_iteration_timeout(self, seconds_from_start=1800):
        # type: (float) -> bool
        """
        Set the ResourceMonitor maximum duration (in seconds) to wait until first scalar/plot is reported.
        If timeout is reached without any reporting, the ResourceMonitor will start reporting machine statistics based
        on seconds from Task start time (instead of based on iteration)

        :param seconds_from_start: Maximum number of seconds to wait for scalar/plot reporting before defaulting
            to machine statistics reporting based on seconds from experiment start time
        :return: True if success
        """
        if not self._resource_monitor:
            return False
        self._resource_monitor.wait_for_first_iteration = seconds_from_start
        self._resource_monitor.max_check_first_iteration = seconds_from_start
        return True

    def execute_remotely(self, queue_name=None, clone=False, exit_process=True):
        # type: (Optional[str], bool, bool) -> Optional[Task]
        """
        If task is running locally (i.e., not by ``clearml-agent``), then clone the Task and enqueue it for remote
        execution; or, stop the execution of the current Task, reset its state, and enqueue it. If ``exit==True``,
        *exit* this process.

        .. note::
            If the task is running remotely (i.e., ``clearml-agent`` is executing it), this call is a no-op
            (i.e., does nothing).

        :param queue_name: The queue name used for enqueueing the task. If ``None``, this call exits the process
            without enqueuing the task.
        :param clone: Clone the Task and execute the newly cloned Task

            The values are:

            - ``True`` - A cloned copy of the Task will be created, and enqueued, instead of this Task.
            - ``False`` - The Task will be enqueued.

        :param exit_process: The function call will leave the calling process at the end

            - ``True`` - Exit the process (exit(0)).
            - ``False`` - Do not exit the process.

            .. warning::

                If ``clone==False``, then ``exit_process`` must be ``True``.

        :return Task: return the task object of the newly generated remotely executing task
        """
        # do nothing, we are running remotely
        if running_remotely() and self.is_main_task():
            return None

        if not self.is_main_task():
            LoggerRoot.get_base_logger().warning(
                "Calling task.execute_remotely is only supported on main Task (created with Task.init)\n"
                "Defaulting to self.enqueue(queue_name={})".format(queue_name)
            )
            if not queue_name:
                raise ValueError("queue_name must be provided")
            enqueue_task = Task.clone(source_task=self) if clone else self
            Task.enqueue(task=enqueue_task, queue_name=queue_name)
            return

        if not clone and not exit_process:
            raise ValueError(
                "clone==False and exit_process==False is not supported. "
                "Task enqueuing itself must exit the process afterwards.")

        # make sure we analyze the process
        if self.status in (Task.TaskStatusEnum.in_progress,):
            if clone:
                # wait for repository detection (5 minutes should be reasonable time to detect all packages)
                self.flush(wait_for_uploads=True)
                if self._logger and not self.__is_subprocess():
                    self._wait_for_repo_detection(timeout=300.)
            else:
                # close ourselves (it will make sure the repo is updated)
                self.close()

        # clone / reset Task
        if clone:
            task = Task.clone(self)
        else:
            task = self
            # check if the server supports enqueueing aborted/stopped Tasks
            if Session.check_min_api_server_version('2.13'):
                self.mark_stopped(force=True)
            else:
                self.reset()

        # enqueue ourselves
        if queue_name:
            Task.enqueue(task, queue_name=queue_name)
            LoggerRoot.get_base_logger().warning(
                'Switching to remote execution, output log page {}'.format(task.get_output_log_web_page()))
        else:
            # Remove the development system tag
            system_tags = [t for t in task.get_system_tags() if t != self._development_tag]
            self.set_system_tags(system_tags)
            # if we leave the Task out there, it makes sense to make it editable.
            self.reset(force=True)

        # leave this process.
        if exit_process:
            LoggerRoot.get_base_logger().warning('Terminating local execution process')
            leave_process(0)

        return task

    def create_function_task(self, func, func_name=None, task_name=None, **kwargs):
        # type: (Callable, Optional[str], Optional[str], **Optional[Any]) -> Optional[Task]
        """
        Create a new task, and call ``func`` with the specified kwargs.
        One can think of this call as remote forking, where the newly created instance is the new Task
        calling the specified func with the appropriate kwargs and leave once the func terminates.
        Notice that a remote executed function cannot create another child remote executed function.

        .. note::
            - Must be called from the main Task, i.e. the one created by Task.init(...)
            - The remote Tasks inherits the environment from the creating Task
            - In the remote Task, the entrypoint is the same as the creating Task
            - In the remote Task, the execution is the same until reaching this function call

        :param func: A function to execute remotely as a single Task.
            On the remote executed Task the entry-point and the environment are copied from this
            calling process, only this function call redirect the the execution flow to the called func,
            alongside the passed arguments
        :param func_name: A unique identifier of the function. Default the function name without the namespace.
            For example Class.foo() becomes 'foo'
        :param task_name: The newly create Task name. Default: the calling Task name + function name
        :param kwargs: name specific arguments for the target function.
            These arguments will appear under the configuration, "Function" section

        :return Task: Return the newly created Task or None if running remotely and execution is skipped
        """
        if not self.is_main_task():
            raise ValueError("Only the main Task object can call create_function_task()")
        if not callable(func):
            raise ValueError("func must be callable")
        if not Session.check_min_api_version('2.9'):
            raise ValueError("Remote function execution is not supported, "
                             "please upgrade to the latest server version")

        func_name = str(func_name or func.__name__).strip()
        if func_name in self._remote_functions_generated:
            raise ValueError("Function name must be unique, a function by the name '{}' "
                             "was already created by this Task.".format(func_name))

        section_name = 'Function'
        tag_name = 'func'
        func_marker = '__func_readonly__'

        # sanitize the dict, leave only basic types that we might want to override later in the UI
        func_params = {k: v for k, v in kwargs.items() if verify_basic_value(v)}
        func_params[func_marker] = func_name

        # do not query if we are running locally, there is no need.
        task_func_marker = self.running_locally() or self.get_parameter('{}/{}'.format(section_name, func_marker))

        # if we are running locally or if we are running remotely but we are not a forked tasks
        # condition explained:
        # (1) running in development mode creates all the forked tasks
        # (2) running remotely but this is not one of the forked tasks (i.e. it is missing the fork tag attribute)
        if self.running_locally() or not task_func_marker:
            self._wait_for_repo_detection(300)
            task = self.clone(self, name=task_name or '{} <{}>'.format(self.name, func_name), parent=self.id)
            task.set_system_tags((task.get_system_tags() or []) + [tag_name])
            task.connect(func_params, name=section_name)
            self._remote_functions_generated[func_name] = task.id
            return task

        # check if we are one of the generated functions and if this is us,
        # if we are not the correct function, not do nothing and leave
        if task_func_marker != func_name:
            self._remote_functions_generated[func_name] = len(self._remote_functions_generated) + 1
            return

        # mark this is us:
        self._remote_functions_generated[func_name] = self.id

        # this is us for sure, let's update the arguments and call the function
        self.connect(func_params, name=section_name)
        func_params.pop(func_marker, None)
        kwargs.update(func_params)
        func(**kwargs)
        # This is it, leave the process
        leave_process(0)

    def wait_for_status(
            self,
            status=(_Task.TaskStatusEnum.completed, _Task.TaskStatusEnum.stopped, _Task.TaskStatusEnum.closed),
            raise_on_status=(_Task.TaskStatusEnum.failed,),
            check_interval_sec=60.,
    ):
        # type: (Iterable[Task.TaskStatusEnum], Optional[Iterable[Task.TaskStatusEnum]], float) -> ()
        """
        Wait for a task to reach a defined status.

        :param status: Status to wait for. Defaults to ('completed', 'stopped', 'closed', )
        :param raise_on_status: Raise RuntimeError if the status of the tasks matches one of these values.
            Defaults to ('failed').
        :param check_interval_sec: Interval in seconds between two checks. Defaults to 60 seconds.

        :raise: RuntimeError if the status is one of {raise_on_status}.
        """
        stopped_status = list(status) + (list(raise_on_status) if raise_on_status else [])
        while self.status not in stopped_status:
            time.sleep(check_interval_sec)

        if raise_on_status and self.status in raise_on_status:
            raise RuntimeError("Task {} has status: {}.".format(self.task_id, self.status))

        # make sure we have the Task object
        self.reload()

    def export_task(self):
        # type: () -> dict
        """
        Export Task's configuration into a dictionary (for serialization purposes).
        A Task can be copied/modified by calling Task.import_task()
        Notice: Export task does not include the tasks outputs, such as results
        (scalar/plots etc.) or Task artifacts/models

        :return: dictionary of the Task's configuration.
        """
        self.reload()
        export_data = self.data.to_dict()
        export_data.pop('last_metrics', None)
        export_data.pop('last_iteration', None)
        export_data.pop('status_changed', None)
        export_data.pop('status_reason', None)
        export_data.pop('status_message', None)
        export_data.get('execution', {}).pop('artifacts', None)
        export_data.get('execution', {}).pop('model', None)
        export_data['project_name'] = self.get_project_name()
        export_data['session_api_version'] = self.session.api_version
        return export_data

    def update_task(self, task_data):
        # type: (dict) -> bool
        """
        Update current task with configuration found on the task_data dictionary.
        See also export_task() for retrieving Task configuration.

        :param task_data: dictionary with full Task configuration
        :return: return True if Task update was successful
        """
        return bool(self.import_task(task_data=task_data, target_task=self, update=True))

    @classmethod
    def import_task(cls, task_data, target_task=None, update=False):
        # type: (dict, Optional[Union[str, Task]], bool) -> Optional[Task]
        """
        Import (create) Task from previously exported Task configuration (see Task.export_task)
        Can also be used to edit/update an existing Task (by passing `target_task` and `update=True`).

        :param task_data: dictionary of a Task's configuration
        :param target_task: Import task_data into an existing Task. Can be either task_id (str) or Task object.
        :param update: If True, merge task_data with current Task configuration.
        :return: return True if Task was imported/updated
        """

        # restore original API version (otherwise, we might not be able to restore the data correctly)
        force_api_version = task_data.get('session_api_version') or None
        original_api_version = Session.api_version
        original_force_max_api_version = Session.force_max_api_version
        if force_api_version:
            Session.force_max_api_version = str(force_api_version)

        if not target_task:
            project_name = task_data.get('project_name') or Task._get_project_name(task_data.get('project', ''))
            target_task = Task.create(project_name=project_name, task_name=task_data.get('name', None))
        elif isinstance(target_task, six.string_types):
            target_task = Task.get_task(task_id=target_task)
        elif not isinstance(target_task, Task):
            raise ValueError(
                "`target_task` must be either Task id (str) or Task object, "
                "received `target_task` type {}".format(type(target_task)))
        target_task.reload()
        cur_data = target_task.data.to_dict()
        cur_data = merge_dicts(cur_data, task_data) if update else dict(**task_data)
        cur_data.pop('id', None)
        cur_data.pop('project', None)
        # noinspection PyProtectedMember
        valid_fields = list(tasks.EditRequest._get_data_props().keys())
        cur_data = dict((k, cur_data[k]) for k in valid_fields if k in cur_data)
        res = target_task._edit(**cur_data)
        if res and res.ok():
            target_task.reload()
        else:
            target_task = None

        # restore current api version, and return a new instance if Task with the current version
        if force_api_version:
            Session.force_max_api_version = original_force_max_api_version
            Session.api_version = original_api_version
            if target_task:
                target_task = Task.get_task(task_id=target_task.id)

        return target_task

    @classmethod
    def import_offline_session(cls, session_folder_zip):
        # type: (str) -> (Optional[str])
        """
        Upload an off line session (execution) of a Task.
        Full Task execution includes repository details, installed packages, artifacts, logs, metric and debug samples.

        :param session_folder_zip: Path to a folder containing the session, or zip-file of the session folder.
        :return: Newly created task ID (str)
        """
        print('ClearML: Importing offline session from {}'.format(session_folder_zip))

        temp_folder = None
        if Path(session_folder_zip).is_file():
            # unzip the file:
            temp_folder = mkdtemp(prefix='clearml-offline-')
            ZipFile(session_folder_zip).extractall(path=temp_folder)
            session_folder_zip = temp_folder

        session_folder = Path(session_folder_zip)
        if not session_folder.is_dir():
            raise ValueError("Could not find the session folder / zip-file {}".format(session_folder))

        try:
            with open((session_folder / cls._offline_filename).as_posix(), 'rt') as f:
                export_data = json.load(f)
        except Exception as ex:
            raise ValueError(
                "Could not read Task object {}: Exception {}".format(session_folder / cls._offline_filename, ex))
        task = cls.import_task(export_data)
        task.mark_started(force=True)
        # fix artifacts
        if task.data.execution.artifacts:
            from . import StorageManager
            # noinspection PyProtectedMember
            offline_folder = os.path.join(export_data.get('offline_folder', ''), 'data/')

            # noinspection PyProtectedMember
            remote_url = task._get_default_report_storage_uri()
            if remote_url and remote_url.endswith('/'):
                remote_url = remote_url[:-1]

            for artifact in task.data.execution.artifacts:
                local_path = artifact.uri.replace(offline_folder, '', 1)
                local_file = session_folder / 'data' / local_path
                if local_file.is_file():
                    remote_path = local_path.replace(
                        '.{}{}'.format(export_data['id'], os.sep), '.{}{}'.format(task.id, os.sep), 1)
                    artifact.uri = '{}/{}'.format(remote_url, remote_path)
                    StorageManager.upload_file(local_file=local_file.as_posix(), remote_url=artifact.uri)
            # noinspection PyProtectedMember
            task._edit(execution=task.data.execution)
        # logs
        TaskHandler.report_offline_session(task, session_folder)
        # metrics
        Metrics.report_offline_session(task, session_folder)
        # print imported results page
        print('ClearML results page: {}'.format(task.get_output_log_web_page()))
        task.mark_completed()
        # close task
        task.close()

        # cleanup
        if temp_folder:
            # noinspection PyBroadException
            try:
                shutil.rmtree(temp_folder)
            except Exception:
                pass

        return task.id

    @classmethod
    def set_credentials(
            cls,
            api_host=None,
            web_host=None,
            files_host=None,
            key=None,
            secret=None,
            store_conf_file=False
    ):
        # type: (Optional[str], Optional[str], Optional[str], Optional[str], Optional[str], bool) -> None
        """
        Set new default **ClearML Server** (backend) host and credentials.

        These credentials will be overridden by either OS environment variables, or the ClearML configuration
        file, ``clearml.conf``.

        .. warning::
           Credentials must be set before initializing a Task object.

        For example, to set credentials for a remote computer:

        .. code-block:: py

            Task.set_credentials(
                api_host='http://localhost:8008', web_host='http://localhost:8080', files_host='http://localhost:8081',
                key='optional_credentials',  secret='optional_credentials'
            )
            task = Task.init('project name', 'experiment name')

        :param str api_host: The API server url. For example, ``host='http://localhost:8008'``
        :param str web_host: The Web server url. For example, ``host='http://localhost:8080'``
        :param str files_host: The file server url. For example, ``host='http://localhost:8081'``
        :param str key: The user key (in the key/secret pair). For example, ``key='thisisakey123'``
        :param str secret: The user secret (in the key/secret pair). For example, ``secret='thisisseceret123'``
        :param bool store_conf_file: If True store the current configuration into the ~/clearml.conf file.
            If the configuration file exists, no change will be made (outputs a warning).
            Not applicable when running remotely (i.e. clearml-agent).
        """
        if api_host:
            Session.default_host = api_host
            if not running_remotely() and not ENV_HOST.get():
                ENV_HOST.set(api_host)
        if web_host:
            Session.default_web = web_host
            if not running_remotely() and not ENV_WEB_HOST.get():
                ENV_WEB_HOST.set(web_host)
        if files_host:
            Session.default_files = files_host
            if not running_remotely() and not ENV_FILES_HOST.get():
                ENV_FILES_HOST.set(files_host)
        if key:
            Session.default_key = key
            if not running_remotely():
                ENV_ACCESS_KEY.set(key)
        if secret:
            Session.default_secret = secret
            if not running_remotely():
                ENV_SECRET_KEY.set(secret)

        if store_conf_file and not running_remotely():
            active_conf_file = get_active_config_file()
            if active_conf_file:
                getLogger().warning(
                    'Could not store credentials in configuration file, '
                    '\'{}\' already exists'.format(active_conf_file))
            else:
                conf = {'api': dict(
                    api_server=Session.default_host,
                    web_server=Session.default_web,
                    files_server=Session.default_files,
                    credentials=dict(access_key=Session.default_key, secret_key=Session.default_secret))}
                with open(get_config_file(), 'wt') as f:
                    lines = json.dumps(conf, indent=4).split('\n')
                    f.write('\n'.join(lines[1:-1]))

    @classmethod
    def debug_simulate_remote_task(cls, task_id, reset_task=False):
        # type: (str, bool) -> ()
        """
        Simulate remote execution of a specified Task.
        This call will simulate the behaviour of your Task as if executed by the ClearML-Agent
        This means configurations will be coming from the backend server into the code
        (the opposite from manual execution, where the backend logs the code arguments)
        Use with care.

        :param task_id: Task ID to simulate, notice that all configuration will be taken from the specified
            Task, regardless of the code initial values, just like it as if executed by ClearML agent
        :param reset_task: If True target Task, is automatically cleared / reset.
        """

        # if we are already running remotely, do nothing
        if running_remotely():
            return

        # verify Task ID exists
        task = Task.get_task(task_id=task_id)
        if not task:
            raise ValueError("Task ID '{}' could not be found".format(task_id))

        if reset_task:
            task.reset(set_started_on_success=False, force=True)

        from .config.remote import override_current_task_id
        from .config.defs import LOG_TO_BACKEND_ENV_VAR
        override_current_task_id(task_id)
        LOG_TO_BACKEND_ENV_VAR.set(True)
        DEBUG_SIMULATE_REMOTE_TASK.set(True)

    @classmethod
    def _create(cls, project_name=None, task_name=None, task_type=TaskTypes.training):
        # type: (Optional[str], Optional[str], Task.TaskTypes) -> Task
        """
        Create a new unpopulated Task (experiment).

        :param str project_name: The name of the project in which the experiment will be created.
            If ``project_name`` is ``None``, and the main execution Task is initialized (see :meth:`Task.init`),
            then the main execution Task's project is used. Otherwise, if the project does
            not exist, it is created. (Optional)
        :param str task_name: The name of Task (experiment).
        :param TaskTypes task_type: The task type.

        :return: The newly created task created.
        """
        if not project_name:
            if not cls.__main_task:
                raise ValueError("Please provide project_name, no global task context found "
                                 "(Task.current_task hasn't been called)")
            project_name = cls.__main_task.get_project_name()

        try:
            task = cls(
                private=cls.__create_protection,
                project_name=project_name,
                task_name=task_name,
                task_type=task_type,
                log_to_backend=False,
                force_create=True,
            )
        except Exception:
            raise
        return task

    def _set_model_config(self, config_text=None, config_dict=None):
        # type: (Optional[str], Optional[Mapping]) -> None
        """
        Set Task model configuration text/dict

        :param config_text: model configuration (unconstrained text string). usually the content
            of a configuration file. If `config_text` is not None, `config_dict` must not be provided.
        :param config_dict: model configuration parameters dictionary.
            If `config_dict` is not None, `config_text` must not be provided.
        """
        # noinspection PyProtectedMember
        design = OutputModel._resolve_config(config_text=config_text, config_dict=config_dict)
        super(Task, self)._set_model_design(design=design)

    def _get_model_config_text(self):
        # type: () -> str
        """
        Get Task model configuration text (before creating an output model)
        When an output model is created it will inherit these properties

        :return: The model config_text (unconstrained text string).
        """
        return super(Task, self).get_model_design()

    def _get_model_config_dict(self):
        # type: () -> Dict
        """
        Get Task model configuration dictionary (before creating an output model)
        When an output model is created it will inherit these properties

        :return: config_dict: model configuration parameters dictionary.
        """
        config_text = self._get_model_config_text()
        # noinspection PyProtectedMember
        return OutputModel._text_to_config_dict(config_text)

    @classmethod
    def _reset_current_task_obj(cls):
        if not cls.__main_task:
            return
        task = cls.__main_task
        cls.__main_task = None
        if task._dev_worker:
            task._dev_worker.unregister()
            task._dev_worker = None

    @classmethod
    def _has_current_task_obj(cls):
        # type: () -> bool
        return bool(cls.__main_task)

    @classmethod
    def _create_dev_task(
            cls, default_project_name, default_task_name, default_task_type, tags,
            reuse_last_task_id, continue_last_task=False, detect_repo=True, auto_connect_streams=True
    ):
        if not default_project_name or not default_task_name:
            # get project name and task name from repository name and entry_point
            result, _ = ScriptInfo.get(create_requirements=False, check_uncommitted=False)
            if not default_project_name:
                # noinspection PyBroadException
                try:
                    parts = result.script['repository'].split('/')
                    default_project_name = (parts[-1] or parts[-2]).replace('.git', '') or 'Untitled'
                except Exception:
                    default_project_name = 'Untitled'
            if not default_task_name:
                # noinspection PyBroadException
                try:
                    default_task_name = os.path.splitext(os.path.basename(result.script['entry_point']))[0]
                except Exception:
                    pass

        # conform reuse_last_task_id and continue_last_task
        if continue_last_task and isinstance(continue_last_task, str):
            reuse_last_task_id = continue_last_task
            continue_last_task = True
        elif isinstance(continue_last_task, int) and continue_last_task is not True:
            # allow initial offset environment override
            continue_last_task = continue_last_task

        if TASK_SET_ITERATION_OFFSET.get() is not None:
            continue_last_task = TASK_SET_ITERATION_OFFSET.get()

        # if we force no task reuse from os environment
        if DEV_TASK_NO_REUSE.get() or not reuse_last_task_id or isinstance(reuse_last_task_id, str):
            default_task = None
        else:
            # if we have a previous session to use, get the task id from it
            default_task = cls.__get_last_used_task_id(
                default_project_name,
                default_task_name,
                default_task_type.value,
            )

        closed_old_task = False
        default_task_id = None
        task = None
        in_dev_mode = not running_remotely()

        if in_dev_mode:
            if isinstance(reuse_last_task_id, str) and reuse_last_task_id:
                default_task_id = reuse_last_task_id
            elif not reuse_last_task_id or not cls.__task_is_relevant(default_task):
                default_task_id = None
            else:
                default_task_id = default_task.get('id') if default_task else None

            if default_task_id:
                try:
                    task = cls(
                        private=cls.__create_protection,
                        task_id=default_task_id,
                        log_to_backend=True,
                    )

                    # instead of resting the previously used task we are continuing the training with it.
                    if task and \
                            (continue_last_task or
                             (isinstance(continue_last_task, int) and not isinstance(continue_last_task, bool))):
                        task.reload()
                        task.mark_started(force=True)
                        # allow to disable the
                        if continue_last_task is True:
                            task.set_initial_iteration(task.get_last_iteration() + 1)
                        else:
                            task.set_initial_iteration(continue_last_task)

                    else:
                        task_tags = task.data.system_tags if hasattr(task.data, 'system_tags') else task.data.tags
                        task_artifacts = task.data.execution.artifacts \
                            if hasattr(task.data.execution, 'artifacts') else None
                        if ((str(task._status) in (
                                str(tasks.TaskStatusEnum.published), str(tasks.TaskStatusEnum.closed)))
                                or task.output_models_id or (cls.archived_tag in task_tags)
                                or (cls._development_tag not in task_tags)
                                or task_artifacts):
                            # If the task is published or closed, we shouldn't reset it so we can't use it in dev mode
                            # If the task is archived, or already has an output model,
                            #  we shouldn't use it in development mode either
                            default_task_id = None
                            task = None
                        else:
                            with task._edit_lock:
                                # from now on, there is no need to reload, we just clear stuff,
                                # this flag will be cleared off once we actually refresh at the end of the function
                                task._reload_skip_flag = True
                                # reset the task, so we can update it
                                task.reset(set_started_on_success=False, force=False)
                                # clear the heaviest stuff first
                                task._clear_task(
                                    system_tags=[cls._development_tag],
                                    comment=make_message('Auto-generated at %(time)s by %(user)s@%(host)s'))

                except (Exception, ValueError):
                    # we failed reusing task, create a new one
                    default_task_id = None

        # create a new task
        if not default_task_id:
            task = cls(
                private=cls.__create_protection,
                project_name=default_project_name,
                task_name=default_task_name,
                task_type=default_task_type,
                log_to_backend=True,
            )
            # no need to reload yet, we clear this before the end of the function
            task._reload_skip_flag = True

        if in_dev_mode:
            # update this session, for later use
            cls.__update_last_used_task_id(default_project_name, default_task_name, default_task_type.value, task.id)
            # set default docker image from env.
            task._set_default_docker_image()

        # mark us as the main Task, there should only be one dev Task at a time.
        if not Task.__main_task:
            Task.__main_task = task

        # mark the task as started
        task.started()
        # reload, making sure we are synced
        task._reload_skip_flag = False
        task.reload()

        # add Task tags
        if tags:
            task.add_tags([tags] if isinstance(tags, str) else tags)

        # force update of base logger to this current task (this is the main logger task)
        logger = task._get_logger(auto_connect_streams=auto_connect_streams)
        if closed_old_task:
            logger.report_text('ClearML Task: Closing old development task id={}'.format(default_task.get('id')))
        # print warning, reusing/creating a task
        if default_task_id and not continue_last_task:
            logger.report_text('ClearML Task: overwriting (reusing) task id=%s' % task.id)
        elif default_task_id and continue_last_task:
            logger.report_text('ClearML Task: continuing previous task id=%s '
                               'Notice this run will not be reproducible!' % task.id)
        else:
            logger.report_text('ClearML Task: created new task id=%s' % task.id)

        # update current repository and put warning into logs
        if detect_repo:
            # noinspection PyBroadException
            try:
                import traceback
                stack = traceback.extract_stack(limit=10)
                # NOTICE WE ARE ALWAYS 3 down from caller in stack!
                for i in range(len(stack) - 1, 0, -1):
                    # look for the Task.init call, then the one above it is the callee module
                    if stack[i].name == 'init':
                        task._calling_filename = os.path.abspath(stack[i - 1].filename)
                        break
            except Exception:
                pass
            if in_dev_mode and cls.__detect_repo_async:
                task._detect_repo_async_thread = threading.Thread(target=task._update_repository)
                task._detect_repo_async_thread.daemon = True
                task._detect_repo_async_thread.start()
            else:
                task._update_repository()

        # make sure we see something in the UI
        thread = threading.Thread(target=LoggerRoot.flush)
        thread.daemon = True
        thread.start()

        return task

    def _get_logger(self, flush_period=NotSet, auto_connect_streams=False):
        # type: (Optional[float], Union[bool, dict]) -> Logger
        """
        get a logger object for reporting based on the task

        :param flush_period: The period of the logger flush.
            If None of any other False value, will not flush periodically.
            If a logger was created before, this will be the new period and
            the old one will be discarded.

        :return: Logger object
        """

        if not self._logger:
            # do not recreate logger after task was closed/quit
            if self._at_exit_called and self._at_exit_called in (True, get_current_thread_id(),):
                raise ValueError("Cannot use Task Logger after task was closed")
            # Get a logger object
            self._logger = Logger(
                private_task=self,
                connect_stdout=(auto_connect_streams is True) or
                               (isinstance(auto_connect_streams, dict) and auto_connect_streams.get('stdout', False)),
                connect_stderr=(auto_connect_streams is True) or
                               (isinstance(auto_connect_streams, dict) and auto_connect_streams.get('stderr', False)),
                connect_logging=isinstance(auto_connect_streams, dict) and auto_connect_streams.get('logging', False),
            )
            # make sure we set our reported to async mode
            # we make sure we flush it in self._at_exit
            self._reporter.async_enable = True
            # if we just created the logger, set default flush period
            if not flush_period or flush_period is self.NotSet:
                flush_period = DevWorker.report_period

        if isinstance(flush_period, (int, float)):
            flush_period = int(abs(flush_period))

        if flush_period is None or isinstance(flush_period, int):
            self._logger.set_flush_period(flush_period)

        return self._logger

    def _connect_output_model(self, model, name=None):
        assert isinstance(model, OutputModel)
        model.connect(self, name=name)
        return model

    def _save_output_model(self, model):
        """
        Deprecated: Save a reference to the connected output model.

        :param model: The connected output model
        """
        # deprecated
        self._connected_output_model = model

    def _reconnect_output_model(self):
        """
        Deprecated: If there is a saved connected output model, connect it again.

        This is needed if the input model is connected after the output model
        is connected, an then we will have to get the model design from the
        input model by reconnecting.
        """
        # Deprecated:
        if self._connected_output_model:
            self.connect(self._connected_output_model)

    def _connect_input_model(self, model, name=None):
        assert isinstance(model, InputModel)
        # we only allow for an input model to be connected once
        # at least until we support multiple input models
        # notice that we do not check the task's input model because we allow task reuse and overwrite
        # add into comment that we are using this model
        comment = self.comment or ''
        if not comment.endswith('\n'):
            comment += '\n'
        comment += 'Using model id: {}'.format(model.id)
        self.set_comment(comment)

        model.connect(self, name)
        return model

    def _connect_argparse(self, parser, args=None, namespace=None, parsed_args=None, name=None):
        # do not allow argparser to connect to jupyter notebook
        # noinspection PyBroadException
        try:
            if 'IPython' in sys.modules:
                # noinspection PyPackageRequirements
                from IPython import get_ipython  # noqa
                ip = get_ipython()
                if ip is not None and 'IPKernelApp' in ip.config:
                    return parser
        except Exception:
            pass

        if self.is_main_task():
            argparser_update_currenttask(self)

        if (parser is None or parsed_args is None) and argparser_parseargs_called():
            # if we have a parser but nor parsed_args, we need to find the parser
            if parser and not parsed_args:
                for _parser, _parsed_args in get_argparser_last_args():
                    if _parser == parser:
                        parsed_args = _parsed_args
                        break
            else:
                # prefer the first argparser (hopefully it is more relevant?!
                for _parser, _parsed_args in get_argparser_last_args():
                    if parser is None:
                        parser = _parser
                    if parsed_args is None and parser == _parser:
                        parsed_args = _parsed_args

        if running_remotely() and (self.is_main_task() or self._is_remote_main_task()):
            self._arguments.copy_to_parser(parser, parsed_args)
        else:
            self._arguments.copy_defaults_from_argparse(
                parser, args=args, namespace=namespace, parsed_args=parsed_args)
        return parser

    def _connect_dictionary(self, dictionary, name=None):
        def _update_args_dict(task, config_dict):
            # noinspection PyProtectedMember
            task._arguments.copy_from_dict(flatten_dictionary(config_dict), prefix=name)

        def _refresh_args_dict(task, config_dict):
            # reread from task including newly added keys
            # noinspection PyProtectedMember
            a_flat_dict = task._arguments.copy_to_dict(flatten_dictionary(config_dict), prefix=name)
            # noinspection PyProtectedMember
            nested_dict = config_dict._to_dict()
            config_dict.clear()
            config_dict.update(nested_from_flat_dictionary(nested_dict, a_flat_dict))

        if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()):
            self._arguments.copy_from_dict(flatten_dictionary(dictionary), prefix=name)
            dictionary = ProxyDictPostWrite(self, _update_args_dict, **dictionary)
        else:
            flat_dict = flatten_dictionary(dictionary)
            flat_dict = self._arguments.copy_to_dict(flat_dict, prefix=name)
            dictionary = nested_from_flat_dictionary(dictionary, flat_dict)
            dictionary = ProxyDictPostWrite(self, _refresh_args_dict, **dictionary)

        return dictionary

    def _connect_task_parameters(self, attr_class, name=None):
        if running_remotely() and (self.is_main_task() or self._is_remote_main_task()):
            parameters = self.get_parameters()
            if not name:
                attr_class.update_from_dict(parameters)
            else:
                attr_class.update_from_dict(
                    dict((k[len(name) + 1:], v) for k, v in parameters.items() if k.startswith('{}/'.format(name))))
        else:
            self.set_parameters(attr_class.to_dict(), __parameters_prefix=name)
        return attr_class

    def _connect_object(self, an_object, name=None):
        def verify_type(key, value):
            if str(key).startswith('_') or not isinstance(value, self._parameters_allowed_types):
                return False
            # verify everything is json able (i.e. basic types)
            try:
                json.dumps(value)
                return True
            except TypeError:
                return False

        a_dict = {k: v for k, v in an_object.__dict__.items() if verify_type(k, v)}
        if running_remotely() and (self.is_main_task() or self._is_remote_main_task()):
            a_dict = self._connect_dictionary(a_dict, name)
            for k, v in a_dict.items():
                if getattr(an_object, k, None) != a_dict[k]:
                    setattr(an_object, k, v)

            return an_object
        else:
            self._connect_dictionary(a_dict, name)
            return an_object

    def _dev_mode_stop_task(self, stop_reason, pid=None):
        # make sure we do not get called (by a daemon thread) after at_exit
        if self._at_exit_called:
            return

        self.log.warning(
            "### TASK STOPPED - USER ABORTED - {} ###".format(
                stop_reason.upper().replace('_', ' ')
            )
        )
        self.flush(wait_for_uploads=True)
        self.stopped(status_reason='USER ABORTED')

        if self._dev_worker:
            self._dev_worker.unregister()

        # NOTICE! This will end the entire execution tree!
        if self.__exit_hook:
            self.__exit_hook.remote_user_aborted = True
        self._kill_all_child_processes(send_kill=False, pid=pid, allow_kill_calling_pid=False)
        time.sleep(2.0)
        self._kill_all_child_processes(send_kill=True, pid=pid, allow_kill_calling_pid=True)
        os._exit(1)  # noqa

    @staticmethod
    def _kill_all_child_processes(send_kill=False, pid=None, allow_kill_calling_pid=True):
        # get current process if pid not provided
        current_pid = os.getpid()
        kill_ourselves = None
        pid = pid or current_pid
        try:
            parent = psutil.Process(pid)
        except psutil.Error:
            # could not find parent process id
            return
        for child in parent.children(recursive=True):
            # kill ourselves last (if we need to)
            if child.pid == current_pid:
                kill_ourselves = child
                continue
            if send_kill:
                child.kill()
            else:
                child.terminate()

        # parent ourselves
        if allow_kill_calling_pid or parent.pid != current_pid:
            if send_kill:
                parent.kill()
            else:
                parent.terminate()

        # kill ourselves if we need to:
        if allow_kill_calling_pid and kill_ourselves:
            if send_kill:
                kill_ourselves.kill()
            else:
                kill_ourselves.terminate()

    def _dev_mode_setup_worker(self):
        if (running_remotely() and not DEBUG_SIMULATE_REMOTE_TASK.get()) \
                or not self.is_main_task() or self._at_exit_called or self._offline_mode:
            return

        if self._dev_worker:
            return self._dev_worker

        self._dev_worker = DevWorker()
        self._dev_worker.register(self)

        logger = self.get_logger()
        flush_period = logger.get_flush_period()
        if not flush_period or flush_period > self._dev_worker.report_period:
            logger.set_flush_period(self._dev_worker.report_period)

    def _wait_for_repo_detection(self, timeout=None):
        # wait for detection repo sync
        if not self._detect_repo_async_thread:
            return
        with self._repo_detect_lock:
            if not self._detect_repo_async_thread:
                return
            # noinspection PyBroadException
            try:
                if self._detect_repo_async_thread.is_alive():
                    # if negative timeout, just kill the thread:
                    if timeout is not None and timeout < 0:
                        from .utilities.lowlevel.threads import kill_thread
                        kill_thread(self._detect_repo_async_thread)
                    else:
                        self.log.info('Waiting for repository detection and full package requirement analysis')
                        self._detect_repo_async_thread.join(timeout=timeout)
                        # because join has no return value
                        if self._detect_repo_async_thread.is_alive():
                            self.log.info('Repository and package analysis timed out ({} sec), '
                                          'giving up'.format(timeout))
                            # done waiting, kill the thread
                            from .utilities.lowlevel.threads import kill_thread
                            kill_thread(self._detect_repo_async_thread)
                        else:
                            self.log.info('Finished repository detection and package analysis')
                self._detect_repo_async_thread = None
            except Exception:
                pass

    def _summary_artifacts(self):
        # signal artifacts upload, and stop daemon
        self._artifacts_manager.stop(wait=True)
        # print artifacts summary (if not empty)
        if self._artifacts_manager.summary:
            self.get_logger().report_text(self._artifacts_manager.summary)

    def _at_exit(self):
        # protect sub-process at_exit (should never happen)
        if self._at_exit_called and self._at_exit_called != get_current_thread_id():
            return

        # make sure we do not try to use events, because Python might deadlock itself.
        # https://bugs.python.org/issue41606
        if self.__is_subprocess():
            BackgroundMonitor.set_at_exit_state(True)

        # shutdown will clear the main, so we have to store it before.
        # is_main = self.is_main_task()
        # fix debugger signal in the middle, catch everything
        try:
            self.__shutdown()
        except:  # noqa
            pass
        # In rare cases we might need to forcefully shutdown the process, currently we should avoid it.
        # if is_main:
        #     # we have to forcefully shutdown if we have forked processes, sometimes they will get stuck
        #     os._exit(self.__exit_hook.exit_code if self.__exit_hook and self.__exit_hook.exit_code else 0)

    def __shutdown(self):
        """
        Will happen automatically once we exit code, i.e. atexit
        :return:
        """
        # protect sub-process at_exit
        if self._at_exit_called:
            is_sub_process = self.__is_subprocess()
            # if we are called twice (signal in the middle of the shutdown),
            _nested_shutdown_call = bool(self._at_exit_called == get_current_thread_id())
            if _nested_shutdown_call and not is_sub_process:
                # if we were called again in the main thread on the main process, let's try again
                # make sure we only do this once
                self._at_exit_called = True
            else:
                # make sure we flush stdout, this is the best we can do.
                if _nested_shutdown_call and self._logger and is_sub_process:
                    # noinspection PyProtectedMember
                    self._logger._close_stdout_handler(wait=True)
                    self._at_exit_called = True
                # if we get here, we should do nothing and leave
                return
        else:
            # from here only a single thread can re-enter
            self._at_exit_called = get_current_thread_id()

        # disable lock on signal callbacks, to avoid deadlocks.
        if self.__exit_hook and self.__exit_hook.signal is not None:
            self.__edit_lock = False

        is_sub_process = self.__is_subprocess()

        # noinspection PyBroadException
        try:
            wait_for_uploads = True
            # first thing mark task as stopped, so we will not end up with "running" on lost tasks
            # if we are running remotely, the daemon will take care of it
            task_status = None
            wait_for_std_log = True
            if (not running_remotely() or DEBUG_SIMULATE_REMOTE_TASK.get()) \
                    and self.is_main_task() and not is_sub_process:
                # check if we crashed, ot the signal is not interrupt (manual break)
                task_status = ('stopped',)
                if self.__exit_hook:
                    is_exception = self.__exit_hook.exception
                    # check if we are running inside a debugger
                    if not is_exception and sys.modules.get('pydevd'):
                        # noinspection PyBroadException
                        try:
                            is_exception = sys.last_type
                        except Exception:
                            pass

                        # check if this is Jupyter interactive session, do not mark as exception
                        if 'IPython' in sys.modules:
                            is_exception = None

                    # only if we have an exception (and not ctrl-break) or signal is not SIGTERM / SIGINT
                    if (is_exception and not isinstance(is_exception, KeyboardInterrupt)
                        and is_exception != KeyboardInterrupt) \
                            or (not self.__exit_hook.remote_user_aborted and
                                self.__exit_hook.signal not in (None, 2, 15)):
                        task_status = (
                            'failed',
                            'Exception {}'.format(is_exception) if is_exception else
                            'Signal {}'.format(self.__exit_hook.signal))
                        wait_for_uploads = False
                    else:
                        wait_for_uploads = (self.__exit_hook.remote_user_aborted or self.__exit_hook.signal is None)
                        if not self.__exit_hook.remote_user_aborted and self.__exit_hook.signal is None and \
                                not is_exception:
                            task_status = ('completed',)
                        else:
                            task_status = ('stopped',)
                            # user aborted. do not bother flushing the stdout logs
                            wait_for_std_log = self.__exit_hook.signal is not None

            # wait for repository detection (if we didn't crash)
            if wait_for_uploads and self._logger:
                # we should print summary here
                self._summary_artifacts()
                # make sure that if we crashed the thread we are not waiting forever
                if not is_sub_process:
                    self._wait_for_repo_detection(timeout=10.)

            # kill the repo thread (negative timeout, do not wait), if it hasn't finished yet.
            if not is_sub_process:
                self._wait_for_repo_detection(timeout=-1)

            # wait for uploads
            print_done_waiting = False
            if wait_for_uploads and (BackendModel.get_num_results() > 0 or
                                     (self.__reporter and self.__reporter.events_waiting())):
                self.log.info('Waiting to finish uploads')
                print_done_waiting = True
            # from here, do not send log in background thread
            if wait_for_uploads:
                self.flush(wait_for_uploads=True)
                # wait until the reporter flush everything
                if self.__reporter:
                    self.__reporter.stop()
                    if self.is_main_task():
                        # notice: this will close the reporting for all the Tasks in the system
                        Metrics.close_async_threads()
                        # notice: this will close the jupyter monitoring
                        ScriptInfo.close()
                if self.is_main_task():
                    # noinspection PyBroadException
                    try:
                        from .storage.helper import StorageHelper
                        StorageHelper.close_async_threads()
                    except Exception:
                        pass

                if print_done_waiting:
                    self.log.info('Finished uploading')
            # elif self._logger:
            #     # noinspection PyProtectedMember
            #     self._logger._flush_stdout_handler()

            # from here, do not check worker status
            if self._dev_worker:
                self._dev_worker.unregister()
                self._dev_worker = None

            # stop resource monitoring
            if self._resource_monitor:
                self._resource_monitor.stop()
                self._resource_monitor = None

            if self._logger:
                self._logger.set_flush_period(None)
                # noinspection PyProtectedMember
                self._logger._close_stdout_handler(wait=wait_for_uploads or wait_for_std_log)

            if not is_sub_process:
                # change task status
                if not task_status:
                    pass
                elif task_status[0] == 'failed':
                    self.mark_failed(status_reason=task_status[1])
                elif task_status[0] == 'completed':
                    self.mark_completed()
                elif task_status[0] == 'stopped':
                    self.stopped()

            # this is so in theory we can close a main task and start a new one
            if self.is_main_task():
                Task.__main_task = None
                Task.__update_master_pid_task(task=None)
        except Exception:
            # make sure we do not interrupt the exit process
            pass

        # make sure we store last task state
        if self._offline_mode and not is_sub_process:
            # noinspection PyBroadException
            try:
                # create zip file
                offline_folder = self.get_offline_mode_folder()
                zip_file = offline_folder.as_posix() + '.zip'
                with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf:
                    for filename in offline_folder.rglob('*'):
                        if filename.is_file():
                            relative_file_name = filename.relative_to(offline_folder).as_posix()
                            zf.write(filename.as_posix(), arcname=relative_file_name)
                print('ClearML Task: Offline session stored in {}'.format(zip_file))
            except Exception:
                pass

        # delete locking object (lock file)
        if self._edit_lock:
            # noinspection PyBroadException
            try:
                del self.__edit_lock
            except Exception:
                pass
            self._edit_lock = None

        # make sure no one will re-enter the shutdown method
        self._at_exit_called = True
        if not is_sub_process and BackgroundMonitor.is_subprocess_enabled():
            BackgroundMonitor.wait_for_sub_process(self)

    @classmethod
    def __register_at_exit(cls, exit_callback, only_remove_signal_and_exception_hooks=False):
        class ExitHooks(object):
            _orig_exit = None
            _orig_exc_handler = None
            remote_user_aborted = False

            def __init__(self, callback):
                self.exit_code = None
                self.exception = None
                self.signal = None
                self._exit_callback = callback
                self._org_handlers = {}
                self._signal_recursion_protection_flag = False
                self._except_recursion_protection_flag = False

            def update_callback(self, callback):
                if self._exit_callback and not six.PY2:
                    # noinspection PyBroadException
                    try:
                        atexit.unregister(self._exit_callback)
                    except Exception:
                        pass
                self._exit_callback = callback
                if callback:
                    self.hook()
                else:
                    # un register int hook
                    if self._orig_exc_handler:
                        sys.excepthook = self._orig_exc_handler
                        self._orig_exc_handler = None
                    for h in self._org_handlers:
                        # noinspection PyBroadException
                        try:
                            signal.signal(h, self._org_handlers[h])
                        except Exception:
                            pass
                    self._org_handlers = {}

            def hook(self):
                if self._orig_exit is None:
                    self._orig_exit = sys.exit
                    sys.exit = self.exit

                if self._orig_exc_handler is None:
                    self._orig_exc_handler = sys.excepthook
                    sys.excepthook = self.exc_handler

                if self._exit_callback:
                    atexit.register(self._exit_callback)

                # TODO: check if sub-process hooks are safe enough, for the time being allow it
                if not self._org_handlers:  # ## and not Task._Task__is_subprocess():
                    if sys.platform == 'win32':
                        catch_signals = [signal.SIGINT, signal.SIGTERM, signal.SIGSEGV, signal.SIGABRT,
                                         signal.SIGILL, signal.SIGFPE]
                    else:
                        catch_signals = [signal.SIGINT, signal.SIGTERM, signal.SIGSEGV, signal.SIGABRT,
                                         signal.SIGILL, signal.SIGFPE, signal.SIGQUIT]
                    for c in catch_signals:
                        # noinspection PyBroadException
                        try:
                            self._org_handlers[c] = signal.getsignal(c)
                            signal.signal(c, self.signal_handler)
                        except Exception:
                            pass

            def exit(self, code=0):
                self.exit_code = code
                self._orig_exit(code)

            def exc_handler(self, exctype, value, traceback, *args, **kwargs):
                if self._except_recursion_protection_flag:
                    # noinspection PyArgumentList
                    return sys.__excepthook__(exctype, value, traceback, *args, **kwargs)

                self._except_recursion_protection_flag = True
                self.exception = value
                if self._orig_exc_handler:
                    # noinspection PyArgumentList
                    ret = self._orig_exc_handler(exctype, value, traceback, *args, **kwargs)
                else:
                    # noinspection PyNoneFunctionAssignment, PyArgumentList
                    ret = sys.__excepthook__(exctype, value, traceback, *args, **kwargs)
                self._except_recursion_protection_flag = False

                return ret

            def signal_handler(self, sig, frame):
                self.signal = sig

                org_handler = self._org_handlers.get(sig)
                signal.signal(sig, org_handler or signal.SIG_DFL)

                # if this is a sig term, we wait until __at_exit is called (basically do nothing)
                if sig == signal.SIGINT:
                    # return original handler result
                    return org_handler if not callable(org_handler) else org_handler(sig, frame)

                if self._signal_recursion_protection_flag:
                    # call original
                    os.kill(os.getpid(), sig)
                    return org_handler if not callable(org_handler) else org_handler(sig, frame)

                self._signal_recursion_protection_flag = True

                # call exit callback
                if self._exit_callback:
                    # noinspection PyBroadException
                    try:
                        self._exit_callback()
                    except Exception:
                        pass

                # remove stdout logger, just in case
                # noinspection PyBroadException
                try:
                    # noinspection PyProtectedMember
                    Logger._remove_std_logger()
                except Exception:
                    pass

                # noinspection PyUnresolvedReferences
                os.kill(os.getpid(), sig)

                self._signal_recursion_protection_flag = False
                # return handler result
                return org_handler if not callable(org_handler) else org_handler(sig, frame)

        # we only remove the signals since this will hang subprocesses
        if only_remove_signal_and_exception_hooks:
            if not cls.__exit_hook:
                return
            if cls.__exit_hook._orig_exc_handler:
                sys.excepthook = cls.__exit_hook._orig_exc_handler
                cls.__exit_hook._orig_exc_handler = None
            for s in cls.__exit_hook._org_handlers:
                # noinspection PyBroadException
                try:
                    signal.signal(s, cls.__exit_hook._org_handlers[s])
                except Exception:
                    pass
            cls.__exit_hook._org_handlers = {}
            return

        if cls.__exit_hook is None:
            # noinspection PyBroadException
            try:
                cls.__exit_hook = ExitHooks(exit_callback)
                cls.__exit_hook.hook()
            except Exception:
                cls.__exit_hook = None
        else:
            cls.__exit_hook.update_callback(exit_callback)

    def _remove_at_exit_callbacks(self):
        self.__register_at_exit(None, only_remove_signal_and_exception_hooks=True)
        atexit.unregister(self.__exit_hook._exit_callback)
        self._at_exit_called = True

    @classmethod
    def __get_task(
            cls,
            task_id=None,  # type: Optional[str]
            project_name=None,  # type: Optional[str]
            task_name=None,  # type: Optional[str]
            include_archived=True,  # type: bool
            tags=None,  # type: Optional[Sequence[str]]
            task_filter=None  # type: Optional[dict]
    ):
        # type: (...) -> Task

        if task_id:
            return cls(private=cls.__create_protection, task_id=task_id, log_to_backend=False)

        if project_name:
            res = cls._send(
                cls._get_default_session(),
                projects.GetAllRequest(
                    name=exact_match_regex(project_name)
                )
            )
            project = get_single_result(entity='project', query=project_name, results=res.response.projects)
        else:
            project = None

        # get default session, before trying to access tasks.Task so that we do not create two sessions.
        session = cls._get_default_session()
        system_tags = 'system_tags' if hasattr(tasks.Task, 'system_tags') else 'tags'
        task_filter = task_filter or {}
        if not include_archived:
            task_filter['system_tags'] = (task_filter.get('system_tags') or []) + ['-{}'.format(cls.archived_tag)]
        if tags:
            task_filter['tags'] = (task_filter.get('tags') or []) + list(tags)
        res = cls._send(
            session,
            tasks.GetAllRequest(
                project=[project.id] if project else None,
                name=exact_match_regex(task_name) if task_name else None,
                only_fields=['id', 'name', 'last_update', system_tags],
                **task_filter
            )
        )
        res_tasks = res.response.tasks
        # if we have more than one result, filter out the 'archived' results
        # notice that if we only have one result we do get the archived one as well.
        if len(res_tasks) > 1:
            filtered_tasks = [t for t in res_tasks if not getattr(t, system_tags, None) or
                              cls.archived_tag not in getattr(t, system_tags, None)]
            # if we did not filter everything (otherwise we have only archived tasks, so we return them)
            if filtered_tasks:
                res_tasks = filtered_tasks

        task = get_single_result(
            entity='task',
            query={k: v for k, v in dict(
                project_name=project_name, task_name=task_name, tags=tags,
                include_archived=include_archived, task_filter=task_filter).items() if v},
            results=res_tasks, raise_on_error=False)
        if not task:
            return None

        return cls(
            private=cls.__create_protection,
            task_id=task.id,
            log_to_backend=False,
        )

    @classmethod
    def __get_tasks(
            cls,
            task_ids=None,  # type: Optional[Sequence[str]]
            project_name=None,  # type: Optional[Union[Sequence[str],str]]
            task_name=None,  # type: Optional[str]
            **kwargs  # type: Any
    ):
        # type: (...) -> List[Task]

        if task_ids:
            if isinstance(task_ids, six.string_types):
                task_ids = [task_ids]
            return [cls(private=cls.__create_protection, task_id=task_id, log_to_backend=False)
                    for task_id in task_ids]

        return [cls(private=cls.__create_protection, task_id=task.id, log_to_backend=False)
                for task in cls._query_tasks(project_name=project_name, task_name=task_name, **kwargs)]

    @classmethod
    def _query_tasks(cls, task_ids=None, project_name=None, task_name=None, **kwargs):
        if not task_ids:
            task_ids = None
        elif isinstance(task_ids, six.string_types):
            task_ids = [task_ids]

        if project_name and isinstance(project_name, str):
            project_names = [project_name]
        else:
            project_names = project_name

        project_ids = []
        if project_names:
            for name in project_names:
                res = cls._send(
                    cls._get_default_session(),
                    projects.GetAllRequest(
                        name=exact_match_regex(name)
                    )
                )
                project = get_single_result(entity='project', query=name, results=res.response.projects)
                if project:
                    project_ids.append(project.id)

        session = cls._get_default_session()
        system_tags = 'system_tags' if hasattr(tasks.Task, 'system_tags') else 'tags'
        only_fields = ['id', 'name', 'last_update', system_tags]

        if kwargs and kwargs.get('only_fields'):
            only_fields = list(set(kwargs.pop('only_fields')) | set(only_fields))

        res = cls._send(
            session,
            tasks.GetAllRequest(
                id=task_ids,
                project=project_ids if project_ids else kwargs.pop('project', None),
                name=task_name if task_name else kwargs.pop('name', None),
                only_fields=only_fields,
                **kwargs
            )
        )

        return res.response.tasks

    @classmethod
    def __get_hash_key(cls, *args):
        def normalize(x):
            return "<{}>".format(x) if x is not None else ""

        return ":".join(map(normalize, args))

    @classmethod
    def __get_last_used_task_id(cls, default_project_name, default_task_name, default_task_type):
        hash_key = cls.__get_hash_key(
            cls._get_api_server(), default_project_name, default_task_name, default_task_type)

        # check if we have a cached task_id we can reuse
        # it must be from within the last 24h and with the same project/name/type
        task_sessions = SessionCache.load_dict(str(cls))

        task_data = task_sessions.get(hash_key)
        if task_data is None:
            return None

        try:
            task_data['type'] = cls.TaskTypes(task_data['type'])
        except (ValueError, KeyError):
            LoggerRoot.get_base_logger().warning(
                "Corrupted session cache entry: {}. "
                "Unsupported task type: {}"
                "Creating a new task.".format(hash_key, task_data['type']),
            )

            return None

        return task_data

    @classmethod
    def __update_last_used_task_id(cls, default_project_name, default_task_name, default_task_type, task_id):
        hash_key = cls.__get_hash_key(
            cls._get_api_server(), default_project_name, default_task_name, default_task_type)

        task_id = str(task_id)
        # update task session cache
        task_sessions = SessionCache.load_dict(str(cls))
        last_task_session = {'time': time.time(), 'project': default_project_name, 'name': default_task_name,
                             'type': default_task_type, 'id': task_id}

        # remove stale sessions
        for k in list(task_sessions.keys()):
            if ((time.time() - task_sessions[k].get('time', 0)) >
                    60 * 60 * cls.__task_id_reuse_time_window_in_hours):
                task_sessions.pop(k)
        # update current session
        task_sessions[hash_key] = last_task_session
        # store
        SessionCache.store_dict(str(cls), task_sessions)

    @classmethod
    def __task_timed_out(cls, task_data):
        return \
            task_data and \
            task_data.get('id') and \
            task_data.get('time') and \
            (time.time() - task_data.get('time')) > (60 * 60 * cls.__task_id_reuse_time_window_in_hours)

    @classmethod
    def __get_task_api_obj(cls, task_id, only_fields=None):
        if not task_id or cls._offline_mode:
            return None

        all_tasks = cls._send(
            cls._get_default_session(),
            tasks.GetAllRequest(id=[task_id], only_fields=only_fields),
        ).response.tasks

        # The task may not exist in environment changes
        if not all_tasks:
            return None

        return all_tasks[0]

    @classmethod
    def __task_is_relevant(cls, task_data):
        """
        Check that a cached task is relevant for reuse.

        A task is relevant for reuse if:
            1. It is not timed out i.e it was last use in the previous 24 hours.
            2. It's name, project and type match the data in the server, so not
               to override user changes made by using the UI.

        :param task_data: A mapping from 'id', 'name', 'project', 'type' keys
            to the task's values, as saved in the cache.

        :return: True, if the task is relevant for reuse. False, if not.
        """
        if not task_data:
            return False

        if cls.__task_timed_out(task_data):
            return False

        task_id = task_data.get('id')

        if not task_id:
            return False

        # noinspection PyBroadException
        try:
            task = cls.__get_task_api_obj(task_id, ('id', 'name', 'project', 'type'))
        except Exception:
            task = None

        if task is None:
            return False

        project_name = None
        if task.project:
            # noinspection PyBroadException
            try:
                project = cls._send(
                    cls._get_default_session(),
                    projects.GetByIdRequest(project=task.project)
                ).response.project

                if project:
                    project_name = project.name
            except Exception:
                pass

        if task_data.get('type') and \
                task_data.get('type') not in (cls.TaskTypes.training, cls.TaskTypes.testing) and \
                not Session.check_min_api_version(2.8):
            print('WARNING: Changing task type to "{}" : '
                  'clearml-server does not support task type "{}", '
                  'please upgrade clearml-server.'.format(cls.TaskTypes.training, task_data['type'].value))
            task_data['type'] = cls.TaskTypes.training

        compares = (
            (task.name, 'name'),
            (project_name, 'project'),
            (task.type, 'type'),
        )

        # compare after casting to string to avoid enum instance issues
        # remember we might have replaced the api version by now, so enums are different
        return all(six.text_type(server_data) == six.text_type(task_data.get(task_data_key))
                   for server_data, task_data_key in compares)

    @classmethod
    def __close_timed_out_task(cls, task_data):
        if not task_data:
            return False

        task = cls.__get_task_api_obj(task_data.get('id'), ('id', 'status'))

        if task is None:
            return False

        stopped_statuses = (
            str(tasks.TaskStatusEnum.stopped),
            str(tasks.TaskStatusEnum.published),
            str(tasks.TaskStatusEnum.publishing),
            str(tasks.TaskStatusEnum.closed),
            str(tasks.TaskStatusEnum.failed),
            str(tasks.TaskStatusEnum.completed),
        )

        if str(task.status) not in stopped_statuses:
            cls._send(
                cls._get_default_session(),
                tasks.StoppedRequest(
                    task=task.id,
                    force=True,
                    status_message="Stopped timed out development task"
                ),
            )

            return True
        return False

    def __getstate__(self):
        # type: () -> dict
        return {'main': self.is_main_task(), 'id': self.id, 'offline': self.is_offline()}

    def __setstate__(self, state):
        if state['main'] and not self.__main_task:
            Task.__forked_proc_main_pid = None
            Task.__update_master_pid_task(task=state['id'])
        if state['offline']:
            Task.set_offline(offline_mode=state['offline'])

        task = Task.init(
            continue_last_task=state['id'],
            auto_connect_frameworks={'detect_repository': False}) \
            if state['main'] else Task.get_task(task_id=state['id'])
        self.__dict__ = task.__dict__


# clearml/task_parameters.py

import six
import attr
from attr import validators


__all__ = ['range_validator', 'param', 'percent_param', 'TaskParameters']


def _canonize_validator(current_validator):
    """
    Convert current_validator to a new list and return it.

    If current_validator is None return an empty list.
    If current_validator is a list, return a copy of it.
    If current_validator is another type of  iterable, return a list version of it.
    If current_validator is a single value, return a one-list containing it.
    """

    if not current_validator:
        return []

    if isinstance(current_validator, (list, tuple)):
        current_validator = list(current_validator)
    else:
        current_validator = [current_validator]

    return current_validator


def range_validator(min_value, max_value):
    """
    A parameter validator that checks range constraint on a parameter.

    :param min_value: The minimum limit of the range, inclusive. None for no minimum limit.
    :param max_value: The maximum limit of the range, inclusive. None for no maximum limit.
    :return: A new range validator.
    """
    def _range_validator(instance, attribute, value):
        if ((min_value is not None) and (value < min_value)) or \
                ((max_value is not None) and (value > max_value)):
            raise ValueError("{} must be in range [{}, {}]".format(attribute.name, min_value, max_value))

    return _range_validator


def param(
        validator=None,
        range=None,
        type=None,
        desc=None,
        metadata=None,
        *args,
        **kwargs
):
    """
    A parameter inside a TaskParameters class.

    See TaskParameters for more information.

    :param validator: A validator or validators list.
        Any validator from attr.validators is applicable.

    :param range: The legal values range of the parameter.
        A tuple (min_limit, max_limit). None for no limitation.

    :param type: The type of the parameter.
        Supported types are int, str and float. None to place no limit of the type

    :param desc: A string description of the parameter, for future use.

    :param metadata: A dictionary metadata of the parameter, for future use.

    :param args: Additional arguments to pass to attr.attrib constructor.
    :param kwargs: Additional keyword arguments to pass to attr.attrib constructor.

    :return: An attr.attrib instance to use with TaskParameters class.

    Warning: Do not create an immutable param using args or kwargs. It will cause
    connect method of the TaskParameters class to fail.
    """

    metadata = metadata or {}
    metadata["desc"] = desc

    validator = _canonize_validator(validator)

    if type:
        validator.append(validators.optional(validators.instance_of(type)))

    if range:
        validator.append(range_validator(*range))

    return attr.ib(validator=validator, type=type, metadata=metadata, *args, **kwargs)


def percent_param(*args, **kwargs):
    """
    A param with type float and range limit (0, 1).
    """
    return param(range=(0, 1), type=float, *args, **kwargs)


class _AttrsMeta(type):
    def __new__(mcs, name, bases, dct):
        new_class = super(_AttrsMeta, mcs).__new__(mcs, name, bases, dct)
        return attr.s(new_class)


@six.add_metaclass(_AttrsMeta)
class TaskParameters(object):
    """
    Base class for task parameters.

    Inherit this class to create a parameter set to connect to a task.

    Usage Example:
    class MyParams(TaskParameters):
        iterations = param(
            type=int,
            desc="Number of iterations to run",
            range=(0, 100000),
        )

        target_accuracy = percent_param(
            desc="The target accuracy of the model",
        )
    """

    def to_dict(self):
        """
        :return: A new dictionary with keys are the parameters names and values
            are the corresponding values.
        """
        return attr.asdict(self)

    def update_from_dict(self, source_dict):
        """
        Update the parameters using values from a dictionary.

        :param source_dict: A dictionary with an entry for each parameter to
            update.
        """
        for key, value in source_dict.items():
            if not hasattr(self, key):
                raise ValueError("Unknown key {} in {} object".format(key, type(self).__name__))

            setattr(self, key, value)

    def connect(self, task):
        """
        Connect to a task.

        When running locally, the task will save the parameters from self.
        When running with a worker, self will be updated according to the task's
        saved parameters.

        :param task: The task to connect to.
        :type task: .Task
        """

        return task.connect(self)


# clearml/version.py

__version__ = '1.2.0rc0'


# clearml/binding/absl_bind.py

""" absl-py FLAGS binding utility functions """
import six

from ..backend_interface.task.args import _Arguments
from ..config import running_remotely


class PatchAbsl(object):
    _original_DEFINE_flag = None
    _original_FLAGS_parse_call = None
    _task = None

    @classmethod
    def update_current_task(cls, current_task):
        cls._task = current_task
        cls._patch_absl()

    @classmethod
    def _patch_absl(cls):
        if cls._original_DEFINE_flag:
            return
        # noinspection PyBroadException
        try:
            from absl.flags import _defines
            if six.PY2:
                cls._original_DEFINE_flag = staticmethod(_defines.DEFINE_flag)
            else:
                cls._original_DEFINE_flag = _defines.DEFINE_flag
            _defines.DEFINE_flag = cls._patched_define_flag
        except Exception:
            # there is no absl
            pass

        try:
            from absl.flags._flagvalues import FlagValues
            if six.PY2:
                cls._original_FLAGS_parse_call = staticmethod(FlagValues.__call__)
            else:
                cls._original_FLAGS_parse_call = FlagValues.__call__
            FlagValues.__call__ = cls._patched_FLAGS_parse_call
        except Exception:
            # there is no absl
            pass

        if cls._original_DEFINE_flag:
            try:
                # if absl was already set, let's update our task params
                from absl import flags
                cls._update_current_flags(flags.FLAGS)
            except Exception:
                # there is no absl
                pass

    @staticmethod
    def _patched_define_flag(*args, **kwargs):
        if not PatchAbsl._task or not PatchAbsl._original_DEFINE_flag:
            if PatchAbsl._original_DEFINE_flag:
                return PatchAbsl._original_DEFINE_flag(*args, **kwargs)
            else:
                return None
        # noinspection PyBroadException
        try:
            flag = args[0] if len(args) >= 1 else None
            module_name = args[2] if len(args) >= 3 else None
            param_name = None
            if flag:
                param_name = ((module_name + _Arguments._prefix_sep) if module_name else '') + flag.name
        except Exception:
            flag = None
            param_name = None

        if running_remotely():
            # noinspection PyBroadException
            try:
                if param_name and flag:
                    param_dict = PatchAbsl._task._arguments.copy_to_dict(
                        {param_name: flag.value}, prefix=_Arguments._prefix_tf_defines)
                    flag.value = param_dict.get(param_name, flag.value)
            except Exception:
                pass
            ret = PatchAbsl._original_DEFINE_flag(*args, **kwargs)
        else:
            if flag and param_name:
                value = flag.value
                PatchAbsl._task.update_parameters({_Arguments._prefix_tf_defines + param_name: value}, )
            ret = PatchAbsl._original_DEFINE_flag(*args, **kwargs)
        return ret

    @staticmethod
    def _patched_FLAGS_parse_call(self, *args, **kwargs):
        ret = PatchAbsl._original_FLAGS_parse_call(self, *args, **kwargs)
        # noinspection PyBroadException
        try:
            PatchAbsl._update_current_flags(self)
        except Exception:
            pass
        return ret

    @classmethod
    def _update_current_flags(cls, FLAGS):
        if not cls._task:
            return
        # noinspection PyBroadException
        try:
            if running_remotely():
                param_dict = dict((k, FLAGS[k].value) for k in FLAGS)
                param_dict = cls._task._arguments.copy_to_dict(param_dict, prefix=_Arguments._prefix_tf_defines)
                for k, v in param_dict.items():
                    # noinspection PyBroadException
                    try:
                        parts = k.split(_Arguments._prefix_sep)
                        k = parts[0]
                        if k in FLAGS:
                            FLAGS[k].value = v
                    except Exception:
                        pass
            else:
                # clear previous parameters
                parameters = dict([(k, FLAGS[k].value) for k in FLAGS])
                # noinspection PyBroadException
                try:
                    descriptions = dict([(k, FLAGS[k].help or None) for k in FLAGS])
                except Exception:
                    descriptions = None
                # noinspection PyBroadException
                try:
                    param_types = dict([(k, FLAGS[k].flag_type() or None) for k in FLAGS])
                except Exception:
                    param_types = None
                cls._task._arguments.copy_from_dict(
                    parameters,
                    prefix=_Arguments._prefix_tf_defines,
                    descriptions=descriptions, param_types=param_types,
                )
        except Exception:
            pass


# clearml/binding/args.py

""" Argparse utilities"""
import sys
from copy import copy

from six import PY2
from argparse import ArgumentParser, Namespace

try:
    from argparse import _SubParsersAction
except ImportError:
    _SubParsersAction = type(None)


class PatchArgumentParser:
    _original_parse_args = None
    _original_parse_known_args = None
    _original_add_subparsers = None
    _original_get_value = None
    _add_subparsers_counter = 0
    _current_task = None
    _calling_current_task = False
    _last_parsed_args = None
    _last_arg_parser = None
    _recursion_guard = False

    @staticmethod
    def add_subparsers(self, **kwargs):
        if 'dest' not in kwargs:
            if kwargs.get('title'):
                kwargs['dest'] = '/' + kwargs['title']
            else:
                PatchArgumentParser._add_subparsers_counter += 1
                kwargs['dest'] = '/subparser%d' % PatchArgumentParser._add_subparsers_counter
        return PatchArgumentParser._original_add_subparsers(self, **kwargs)

    @staticmethod
    def parse_args(self, args=None, namespace=None):
        if PatchArgumentParser._recursion_guard:
            return {} if not PatchArgumentParser._original_parse_args else \
                PatchArgumentParser._original_parse_args(self, args=args, namespace=namespace)

        PatchArgumentParser._recursion_guard = True
        try:
            result = PatchArgumentParser._patched_parse_args(
                PatchArgumentParser._original_parse_args, self, args=args, namespace=namespace)
        finally:
            PatchArgumentParser._recursion_guard = False
        return result

    @staticmethod
    def parse_known_args(self, args=None, namespace=None):
        if PatchArgumentParser._recursion_guard:
            return {} if not PatchArgumentParser._original_parse_args else \
                PatchArgumentParser._original_parse_known_args(self, args=args, namespace=namespace)

        PatchArgumentParser._recursion_guard = True
        try:
            result = PatchArgumentParser._patched_parse_args(
                PatchArgumentParser._original_parse_known_args, self, args=args, namespace=namespace)
        finally:
            PatchArgumentParser._recursion_guard = False
        return result

    @staticmethod
    def _patched_parse_args(original_parse_fn, self, args=None, namespace=None):
        current_task = PatchArgumentParser._current_task
        # if we are running remotely, we always have a task id, so we better patch the argparser as soon as possible.
        if not current_task:
            from ..config import running_remotely, get_remote_task_id
            if running_remotely():
                # this will cause the current_task() to set PatchArgumentParser._current_task
                from clearml import Task
                # noinspection PyBroadException
                try:
                    current_task = Task.get_task(task_id=get_remote_task_id())
                    # make sure we do not store back the values
                    # (we will do that when we actually call parse args)
                    # this will make sure that if we have args we should not track we know them
                    # noinspection PyProtectedMember
                    current_task._arguments.exclude_parser_args({'*': True})
                except Exception:
                    pass
        # automatically connect to current task:
        if current_task:
            from ..config import running_remotely

            if PatchArgumentParser._calling_current_task:
                # if we are here and running remotely by now we should try to parse the arguments
                parsed_args = None
                if original_parse_fn:
                    parsed_args = original_parse_fn(self, args=args, namespace=namespace)
                    PatchArgumentParser._add_last_parsed_args(self, parsed_args)
                return parsed_args or PatchArgumentParser._last_parsed_args[-1]

            PatchArgumentParser._calling_current_task = True
            # Store last instance and result
            PatchArgumentParser._add_last_arg_parser(self)
            parsed_args = parsed_args_str = None
            # parse if we are running in dev mode
            if not running_remotely() and original_parse_fn:
                parsed_args = original_parse_fn(self, args=args, namespace=namespace)
                parsed_args_str = PatchArgumentParser._add_last_parsed_args(self, parsed_args)

            # noinspection PyBroadException
            try:
                # sync to/from task
                # noinspection PyProtectedMember
                current_task._connect_argparse(
                    self, args=args, namespace=namespace,
                    parsed_args=parsed_args_str[0] if isinstance(parsed_args_str, tuple) else parsed_args_str
                )
            except Exception:
                pass

            # sync back and parse
            if running_remotely():
                if original_parse_fn:
                    # if we are running python2 check if we have subparsers,
                    # if we do we need to patch the args, because there is no default subparser
                    if PY2:
                        import itertools

                        def _get_sub_parsers_defaults(subparser, prev=[]):
                            actions_grp = [v._actions for v in subparser.choices.values()] if isinstance(
                                subparser, _SubParsersAction) else [subparser._actions]
                            _sub_parsers_defaults = [[subparser]] if hasattr(
                                subparser, 'default') and subparser.default else []
                            for actions in actions_grp:
                                _sub_parsers_defaults += [_get_sub_parsers_defaults(v, prev)
                                                          for v in actions if isinstance(v, _SubParsersAction) and
                                                          hasattr(v, 'default') and v.default]

                            return list(itertools.chain.from_iterable(_sub_parsers_defaults))

                        sub_parsers_defaults = _get_sub_parsers_defaults(self)
                        if sub_parsers_defaults:
                            if args is None:
                                # args default to the system args
                                import sys as _sys
                                args = _sys.argv[1:]
                            else:
                                args = list(args)
                            # make sure we append the subparsers
                            for a in sub_parsers_defaults:
                                if a.default not in args:
                                    args.append(a.default)

                    parsed_args = original_parse_fn(self, args=args, namespace=namespace)
                    PatchArgumentParser._add_last_parsed_args(self, parsed_args)
                else:
                    # we should never get here
                    parsed_args = parsed_args_str or {}
                    PatchArgumentParser._add_last_parsed_args(self, parsed_args)

            PatchArgumentParser._calling_current_task = False
            return parsed_args

        # Store last instance and result
        PatchArgumentParser._add_last_arg_parser(self)
        parsed_args = {} if not original_parse_fn else original_parse_fn(self, args=args, namespace=namespace)
        PatchArgumentParser._add_last_parsed_args(self, parsed_args)
        return parsed_args

    @staticmethod
    def _add_last_parsed_args(parser, parsed_args):
        if hasattr(parser, '_parsed_arg_string_lookup'):
            if isinstance(parsed_args, tuple):
                parsed_args_namespace = copy(parsed_args[0])
                parsed_args = (parsed_args_namespace, parsed_args[1])
            else:
                parsed_args_namespace = copy(parsed_args)
                parsed_args = (parsed_args_namespace, [])

            # cast arguments in parsed_args_namespace entries to str
            if parsed_args_namespace and isinstance(parsed_args_namespace, Namespace):
                for k, v in parser._parsed_arg_string_lookup.items():  # noqa
                    if hasattr(parsed_args_namespace, k):
                        if isinstance(getattr(parsed_args_namespace, k, None), list) and not isinstance(v, list):
                            v = [v]
                        setattr(
                            parsed_args_namespace, k,
                            v if isinstance(v, list) else str(v)
                        )

        if not PatchArgumentParser._last_parsed_args or parsed_args not in PatchArgumentParser._last_parsed_args:
            PatchArgumentParser._last_parsed_args = (PatchArgumentParser._last_parsed_args or []) + [parsed_args]
        return parsed_args

    @staticmethod
    def _add_last_arg_parser(a_argparser):
        PatchArgumentParser._last_arg_parser = (PatchArgumentParser._last_arg_parser or []) + [a_argparser]

    @staticmethod
    def _get_value(self, action, arg_string):
        if not hasattr(self, '_parsed_arg_string_lookup'):
            setattr(self, '_parsed_arg_string_lookup', dict())
        k = str(action.dest)
        if k not in self._parsed_arg_string_lookup:
            self._parsed_arg_string_lookup[k] = arg_string
        else:
            self._parsed_arg_string_lookup[k] = \
                (self._parsed_arg_string_lookup[k]
                 if isinstance(self._parsed_arg_string_lookup[k], list)
                 else [self._parsed_arg_string_lookup[k]]) + [arg_string]
        return PatchArgumentParser._original_get_value(self, action, arg_string)


def patch_argparse():
    # make sure we only patch once
    if not sys.modules.get('argparse') or hasattr(sys.modules['argparse'].ArgumentParser, '_parse_args_patched'):
        return
    # mark patched argparse
    sys.modules['argparse'].ArgumentParser._parse_args_patched = True
    # patch argparser
    PatchArgumentParser._original_parse_args = sys.modules['argparse'].ArgumentParser.parse_args
    PatchArgumentParser._original_parse_known_args = sys.modules['argparse'].ArgumentParser.parse_known_args
    PatchArgumentParser._original_add_subparsers = sys.modules['argparse'].ArgumentParser.add_subparsers
    sys.modules['argparse'].ArgumentParser.parse_args = PatchArgumentParser.parse_args
    sys.modules['argparse'].ArgumentParser.parse_known_args = PatchArgumentParser.parse_known_args
    sys.modules['argparse'].ArgumentParser.add_subparsers = PatchArgumentParser.add_subparsers
    if hasattr(sys.modules['argparse'].ArgumentParser, '_get_value'):
        PatchArgumentParser._original_get_value = sys.modules['argparse'].ArgumentParser._get_value
        sys.modules['argparse'].ArgumentParser._get_value = PatchArgumentParser._get_value


# Notice! we are patching argparser, so we know if someone parsed arguments before connecting to task
patch_argparse()


def call_original_argparser(self, args=None, namespace=None):
    if PatchArgumentParser._original_parse_args:
        return PatchArgumentParser._original_parse_args(self, args=args, namespace=namespace)


def argparser_parseargs_called():
    return PatchArgumentParser._last_arg_parser is not None


def argparser_update_currenttask(task):
    PatchArgumentParser._current_task = task


def get_argparser_last_args():
    if not PatchArgumentParser._last_arg_parser or not PatchArgumentParser._last_parsed_args:
        return []

    return [(parser, args[0] if isinstance(args, tuple) else args)
            for parser, args in zip(PatchArgumentParser._last_arg_parser, PatchArgumentParser._last_parsed_args)]


def add_params_to_parser(parser, params):
    assert isinstance(parser, ArgumentParser)
    assert isinstance(params, dict)

    def get_type_details(v):
        for t in (int, float, str):
            try:
                _value = t(v)
                return t, _value
            except ValueError:
                continue

    # AJB temporary protection from ui problems sending empty dicts
    params.pop('', None)

    for param, value in params.items():
        _type, type_value = get_type_details(value)
        parser.add_argument('--%s' % param, type=_type, default=type_value)
    return parser


# clearml/binding/artifacts.py

import json
import mimetypes
import os
import pickle
from six.moves.urllib.parse import quote
from copy import deepcopy
from datetime import datetime
from multiprocessing import RLock, Event
from multiprocessing.pool import ThreadPool
from tempfile import mkdtemp, mkstemp
from threading import Thread
from time import time
from zipfile import ZipFile, ZIP_DEFLATED

import six
from PIL import Image
from pathlib2 import Path
from six.moves.urllib.parse import urlparse
from typing import Dict, Union, Optional, Any, Sequence

from ..backend_api import Session
from ..backend_api.services import tasks
from ..backend_interface.metrics.events import UploadEvent
from ..debugging.log import LoggerRoot
from ..storage.helper import remote_driver_schemes
from ..storage.util import sha256sum, format_size, get_common_path
from ..utilities.proxy_object import LazyEvalWrapper

try:
    import pandas as pd
    DataFrame = pd.DataFrame
except ImportError:
    pd = None
    DataFrame = None
try:
    import numpy as np
except ImportError:
    np = None
try:
    from pathlib import Path as pathlib_Path
except ImportError:
    pathlib_Path = None


class Artifact(object):
    """
    Read-Only Artifact object
    """

    @property
    def url(self):
        # type: () -> str
        """
        :return: The URL of uploaded artifact.
        """
        return self._url

    @property
    def name(self):
        # type: () -> str
        """
        :return: The name of artifact.
        """
        return self._name

    @property
    def size(self):
        # type: () -> int
        """
        :return: The size in bytes of artifact.
        """
        return self._size

    @property
    def type(self):
        # type: () -> str
        """
        :return: The type (str) of of artifact.
        """
        return self._type

    @property
    def mode(self):
        # type: () -> Union["input", "output"]  # noqa: F821
        """
        :return: The mode (str) of of artifact: "input" or "output".
        """
        return self._mode

    @property
    def hash(self):
        # type: () -> str
        """
        :return: SHA2 hash (str) of of artifact content.
        """
        return self._hash

    @property
    def timestamp(self):
        # type: () -> datetime
        """
        :return: Timestamp (datetime) of uploaded artifact.
        """
        return self._timestamp

    @property
    def metadata(self):
        # type: () -> Optional[Dict[str, str]]
        """
        :return: Key/Value dictionary attached to artifact.
        """
        return self._metadata

    @property
    def preview(self):
        # type: () -> str
        """
        :return: A string (str) representation of the artifact.
        """
        return self._preview

    def __init__(self, artifact_api_object):
        """
        construct read-only object from api artifact object

        :param tasks.Artifact artifact_api_object:
        """
        self._name = artifact_api_object.key
        self._size = artifact_api_object.content_size
        self._type = artifact_api_object.type
        self._mode = artifact_api_object.mode
        self._url = artifact_api_object.uri
        self._hash = artifact_api_object.hash
        self._timestamp = datetime.fromtimestamp(artifact_api_object.timestamp or 0)
        self._metadata = dict(artifact_api_object.display_data) if artifact_api_object.display_data else {}
        self._preview = artifact_api_object.type_data.preview if artifact_api_object.type_data else None
        self._object = None

    def get(self, force_download=False):
        # type: (bool) -> Any
        """
        Return an object constructed from the artifact file

        Currently supported types: Numpy.array, pandas.DataFrame, PIL.Image, dict (json)
        All other types will return a pathlib2.Path object pointing to a local copy of the artifacts file (or directory)

        :param bool force_download: download file from remote even if exists in local cache
        :return: One of the following objects Numpy.array, pandas.DataFrame, PIL.Image, dict (json), or pathlib2.Path.
        """
        if self._object:
            return self._object

        local_file = self.get_local_copy(raise_on_error=True, force_download=force_download)

        # noinspection PyProtectedMember
        if self.type == 'numpy' and np:
            self._object = np.load(local_file)[self.name]
        elif self.type == Artifacts._pd_artifact_type and pd:
            self._object = pd.read_csv(local_file)
        elif self.type == 'pandas' and pd:
            self._object = pd.read_csv(local_file, index_col=[0])
        elif self.type == 'image':
            self._object = Image.open(local_file)
        elif self.type == 'JSON':
            with open(local_file, 'rt') as f:
                self._object = json.load(f)
        elif self.type == 'string':
            with open(local_file, 'rt') as f:
                self._object = f.read()
        elif self.type == 'pickle':
            with open(local_file, 'rb') as f:
                self._object = pickle.load(f)

        local_file = Path(local_file)

        if self._object is None:
            self._object = local_file

        return self._object

    def get_local_copy(self, extract_archive=True, raise_on_error=False, force_download=False):
        # type: (bool, bool, bool) -> str
        """
        :param bool extract_archive: If True and artifact is of type 'archive' (compressed folder)
            The returned path will be a temporary folder containing the archive content
        :param bool raise_on_error: If True and the artifact could not be downloaded,
            raise ValueError, otherwise return None on failure and output log warning.
        :param bool force_download: download file from remote even if exists in local cache
        :raise: Raises error if local copy not found.
        :return: A local path to a downloaded copy of the artifact.
        """
        from clearml.storage import StorageManager
        local_copy = StorageManager.get_local_copy(
            remote_url=self.url,
            extract_archive=extract_archive and self.type == 'archive',
            name=self.name,
            force_download=force_download
        )
        if raise_on_error and local_copy is None:
            raise ValueError(
                "Could not retrieve a local copy of artifact {}, failed downloading {}".format(self.name, self.url))

        return local_copy

    def __repr__(self):
        return str({'name': self.name, 'size': self.size, 'type': self.type, 'mode': self.mode, 'url': self.url,
                    'hash': self.hash, 'timestamp': self.timestamp,
                    'metadata': self.metadata, 'preview': self.preview, })


class Artifacts(object):
    max_preview_size_bytes = 65536

    _flush_frequency_sec = 300.
    # notice these two should match
    _save_format = '.csv.gz'
    _compression = 'gzip'
    # hashing constants
    _hash_block_size = 65536
    _pd_artifact_type = 'data-audit-table'

    class _ProxyDictWrite(dict):
        """ Dictionary wrapper that updates an arguments instance on any item set in the dictionary """

        def __init__(self, artifacts_manager, *args, **kwargs):
            super(Artifacts._ProxyDictWrite, self).__init__(*args, **kwargs)
            self._artifacts_manager = artifacts_manager
            # list of artifacts we should not upload (by name & weak-reference)
            self.artifact_metadata = {}
            # list of hash columns to calculate uniqueness for the artifacts
            self.artifact_hash_columns = {}

        def __setitem__(self, key, value):
            # check that value is of type pandas
            if pd and isinstance(value, pd.DataFrame):
                super(Artifacts._ProxyDictWrite, self).__setitem__(key, value)

                if self._artifacts_manager:
                    self._artifacts_manager.flush()
            else:
                raise ValueError('Artifacts currently support pandas.DataFrame objects only')

        def unregister_artifact(self, name):
            self.artifact_metadata.pop(name, None)
            self.pop(name, None)

        def add_metadata(self, name, metadata):
            self.artifact_metadata[name] = deepcopy(metadata)

        def get_metadata(self, name):
            return self.artifact_metadata.get(name)

        def add_hash_columns(self, artifact_name, hash_columns):
            self.artifact_hash_columns[artifact_name] = hash_columns

        def get_hash_columns(self, artifact_name):
            return self.artifact_hash_columns.get(artifact_name)

    @property
    def registered_artifacts(self):
        # type: () -> Dict[str, Artifact]
        return self._artifacts_container

    @property
    def summary(self):
        # type: () -> str
        return self._summary

    def __init__(self, task):
        self._task = task
        # notice the double link, this is important since the Artifact
        # dictionary needs to signal the Artifacts base on changes
        self._artifacts_container = self._ProxyDictWrite(self)
        self._last_artifacts_upload = {}
        self._unregister_request = set()
        self._thread = None
        self._flush_event = Event()
        self._exit_flag = False
        self._summary = ''
        self._temp_folder = []
        self._task_artifact_list = []
        self._task_edit_lock = RLock()
        self._storage_prefix = None

    def register_artifact(self, name, artifact, metadata=None, uniqueness_columns=True):
        # type: (str, DataFrame, Optional[dict], Union[bool, Sequence[str]]) -> ()
        """
        :param str name: name of the artifacts. Notice! it will override previous artifacts if name already exists.
        :param pandas.DataFrame artifact: artifact object, supported artifacts object types: pandas.DataFrame
        :param dict metadata: dictionary of key value to store with the artifact (visible in the UI)
        :param list uniqueness_columns: list of columns for artifact uniqueness comparison criteria. The default value
            is True, which equals to all the columns (same as artifact.columns).
        """
        # currently we support pandas.DataFrame (which we will upload as csv.gz)
        if name in self._artifacts_container:
            LoggerRoot.get_base_logger().info('Register artifact, overwriting existing artifact \"{}\"'.format(name))
        self._artifacts_container.add_hash_columns(
            name, list(artifact.columns if uniqueness_columns is True else uniqueness_columns)
        )
        self._artifacts_container[name] = artifact
        if metadata:
            self._artifacts_container.add_metadata(name, metadata)

    def unregister_artifact(self, name):
        # type: (str) -> ()
        # Remove artifact from the watch list
        self._unregister_request.add(name)
        self.flush()

    def upload_artifact(self, name, artifact_object=None, metadata=None, preview=None,
                        delete_after_upload=False, auto_pickle=True, wait_on_upload=False):
        # type: (str, Optional[object], Optional[dict], Optional[str], bool, bool, bool) -> bool
        if not Session.check_min_api_version('2.3'):
            LoggerRoot.get_base_logger().warning('Artifacts not supported by your ClearML-server version, '
                                                 'please upgrade to the latest server version')
            return False

        if name in self._artifacts_container:
            raise ValueError("Artifact by the name of {} is already registered, use register_artifact".format(name))

        # cast preview to string
        if preview is not None and not (isinstance(preview, bool) and preview is False):
            preview = str(preview)

        # evaluate lazy proxy object
        if isinstance(artifact_object, LazyEvalWrapper):
            # noinspection PyProtectedMember
            artifact_object = LazyEvalWrapper._load_object(artifact_object)

        pathlib_types = (Path, pathlib_Path,) if pathlib_Path is not None else (Path,)
        local_filename = None

        # try to convert string Path object (it might reference a file/folder)
        # dont not try to serialize long texts.
        if isinstance(artifact_object, six.string_types) and artifact_object and len(artifact_object) < 2048:
            # noinspection PyBroadException
            try:
                artifact_path = Path(artifact_object)
                if artifact_path.exists():
                    artifact_object = artifact_path
                elif '*' in artifact_object or '?' in artifact_object:
                    # hackish, detect wildcard in tr files
                    folder = Path('').joinpath(*artifact_path.parts[:-1])
                    if folder.is_dir() and folder.parts:
                        wildcard = artifact_path.parts[-1]
                        if list(Path(folder).rglob(wildcard)):
                            artifact_object = artifact_path
            except Exception:
                pass

        store_as_pickle = False
        artifact_type_data = tasks.ArtifactTypeData()
        artifact_type_data.preview = ''
        override_filename_in_uri = None
        override_filename_ext_in_uri = None
        uri = None
        if np and isinstance(artifact_object, np.ndarray):
            artifact_type = 'numpy'
            artifact_type_data.content_type = 'application/numpy'
            artifact_type_data.preview = preview or str(artifact_object.__repr__())
            override_filename_ext_in_uri = '.npz'
            override_filename_in_uri = name + override_filename_ext_in_uri
            fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
            os.close(fd)
            np.savez_compressed(local_filename, **{name: artifact_object})
            delete_after_upload = True
        elif pd and isinstance(artifact_object, pd.DataFrame):
            artifact_type = 'pandas'
            artifact_type_data.content_type = 'text/csv'
            artifact_type_data.preview = preview or str(artifact_object.__repr__())
            override_filename_ext_in_uri = self._save_format
            override_filename_in_uri = name
            fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
            os.close(fd)
            artifact_object.to_csv(local_filename, compression=self._compression)
            delete_after_upload = True
        elif isinstance(artifact_object, Image.Image):
            artifact_type = 'image'
            artifact_type_data.content_type = 'image/png'
            desc = str(artifact_object.__repr__())
            artifact_type_data.preview = preview or desc[1:desc.find(' at ')]
            override_filename_ext_in_uri = '.png'
            override_filename_in_uri = name + override_filename_ext_in_uri
            fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
            os.close(fd)
            artifact_object.save(local_filename)
            delete_after_upload = True
        elif isinstance(artifact_object, dict):
            artifact_type = 'JSON'
            artifact_type_data.content_type = 'application/json'
            # noinspection PyBroadException
            try:
                json_text = json.dumps(artifact_object, sort_keys=True, indent=4)
            except Exception:
                if not auto_pickle:
                    raise
                LoggerRoot.get_base_logger().warning(
                    "JSON serialization of artifact \'{}\' failed, reverting to pickle".format(name))
                store_as_pickle = True
                json_text = None

            if json_text is not None:
                override_filename_ext_in_uri = '.json'
                override_filename_in_uri = name + override_filename_ext_in_uri
                fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
                os.write(fd, bytes(json_text.encode()))
                os.close(fd)
                preview = preview or json_text
                if len(preview) < self.max_preview_size_bytes:
                    artifact_type_data.preview = preview
                else:
                    artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format(
                        self.max_preview_size_bytes//1024, preview[:self.max_preview_size_bytes]
                    )

                delete_after_upload = True
        elif isinstance(artifact_object, pathlib_types):
            # check if single file
            artifact_object = Path(artifact_object)

            artifact_object.expanduser().absolute()
            # noinspection PyBroadException
            try:
                create_zip_file = not artifact_object.is_file()
            except Exception:  # Hack for windows pathlib2 bug, is_file isn't valid.
                create_zip_file = True
            else:  # We assume that this is not Windows os
                if artifact_object.is_dir():
                    # change to wildcard
                    artifact_object /= '*'

            if create_zip_file:
                folder = Path('').joinpath(*artifact_object.parts[:-1])
                if not folder.is_dir() or not folder.parts:
                    raise ValueError("Artifact file/folder '{}' could not be found".format(
                        artifact_object.as_posix()))

                wildcard = artifact_object.parts[-1]
                files = list(Path(folder).rglob(wildcard))
                override_filename_ext_in_uri = '.zip'
                override_filename_in_uri = folder.parts[-1] + override_filename_ext_in_uri
                fd, zip_file = mkstemp(
                    prefix=quote(folder.parts[-1], safe="") + '.', suffix=override_filename_ext_in_uri
                )
                try:
                    artifact_type_data.content_type = 'application/zip'
                    archive_preview = 'Archive content {}:\n'.format(artifact_object.as_posix())

                    with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf:
                        for filename in sorted(files):
                            if filename.is_file():
                                relative_file_name = filename.relative_to(folder).as_posix()
                                archive_preview += '{} - {}\n'.format(
                                    relative_file_name, format_size(filename.stat().st_size))
                                zf.write(filename.as_posix(), arcname=relative_file_name)
                except Exception as e:
                    # failed uploading folder:
                    LoggerRoot.get_base_logger().warning('Exception {}\nFailed zipping artifact folder {}'.format(
                        folder, e))
                    return False
                finally:
                    os.close(fd)
                artifact_type_data.preview = preview or archive_preview
                artifact_object = zip_file
                artifact_type = 'archive'
                artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0]
                local_filename = artifact_object
                delete_after_upload = True
            else:
                if not artifact_object.is_file():
                    raise ValueError("Artifact file '{}' could not be found".format(artifact_object.as_posix()))

                override_filename_in_uri = artifact_object.parts[-1]
                artifact_type_data.preview = preview or '{} - {}\n'.format(
                    artifact_object, format_size(artifact_object.stat().st_size))
                artifact_object = artifact_object.as_posix()
                artifact_type = 'custom'
                artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0]
                local_filename = artifact_object
        elif isinstance(artifact_object, (list, tuple)) and all(isinstance(p, pathlib_types) for p in artifact_object):
            # find common path if exists
            list_files = [Path(p) for p in artifact_object]
            override_filename_ext_in_uri = '.zip'
            override_filename_in_uri = quote(name, safe="") + override_filename_ext_in_uri
            common_path = get_common_path(list_files)
            fd, zip_file = mkstemp(
                prefix='artifact_folder.', suffix=override_filename_ext_in_uri
            )
            try:
                artifact_type_data.content_type = 'application/zip'
                archive_preview = 'Archive content:\n'

                with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf:
                    for filename in sorted(list_files):
                        if filename.is_file():
                            relative_file_name = filename.relative_to(Path(common_path)).as_posix() \
                                if common_path else filename.as_posix()
                            archive_preview += '{} - {}\n'.format(
                                relative_file_name, format_size(filename.stat().st_size))
                            zf.write(filename.as_posix(), arcname=relative_file_name)
                        else:
                            LoggerRoot.get_base_logger().warning(
                                "Failed zipping artifact file '{}', file not found!".format(filename.as_posix()))
            except Exception as e:
                # failed uploading folder:
                LoggerRoot.get_base_logger().warning('Exception {}\nFailed zipping artifact files {}'.format(
                    artifact_object, e))
                return False
            finally:
                os.close(fd)
            artifact_type_data.preview = preview or archive_preview
            artifact_object = zip_file
            artifact_type = 'archive'
            artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0]
            local_filename = artifact_object
            delete_after_upload = True
        elif (
                isinstance(artifact_object, six.string_types) and len(artifact_object) < 4096
                and urlparse(artifact_object).scheme in remote_driver_schemes
        ):
            # we should not upload this, just register
            local_filename = None
            uri = artifact_object
            artifact_type = 'custom'
            artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0]
            if preview:
                artifact_type_data.preview = preview
        elif isinstance(artifact_object, six.string_types):
            # if we got here, we should store it as text file.
            artifact_type = 'string'
            artifact_type_data.content_type = 'text/plain'
            if preview:
                artifact_type_data.preview = preview
            elif len(artifact_object) < self.max_preview_size_bytes:
                artifact_type_data.preview = artifact_object
            else:
                artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format(
                    self.max_preview_size_bytes//1024, artifact_object[:self.max_preview_size_bytes]
                )
            if artifact_object:
                delete_after_upload = True
                override_filename_ext_in_uri = '.txt'
                override_filename_in_uri = name + override_filename_ext_in_uri
                fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
                os.close(fd)
                # noinspection PyBroadException
                try:
                    with open(local_filename, 'wt') as f:
                        f.write(artifact_object)
                except Exception:
                    # cleanup and raise exception
                    os.unlink(local_filename)
                    raise
        elif artifact_object is None or (isinstance(artifact_object, str) and artifact_object == ""):
            artifact_type = ''
            store_as_pickle = False
        elif auto_pickle:
            # revert to pickling the object
            store_as_pickle = True
        else:
            raise ValueError("Artifact type {} not supported".format(type(artifact_object)))

        # revert to serializing the object with pickle
        if store_as_pickle:
            # if we are here it means we do not know what to do with the object, so we serialize it with pickle.
            artifact_type = 'pickle'
            artifact_type_data.content_type = 'application/pickle'
            # noinspection PyBroadException
            try:
                artifact_type_data.preview = preview or str(artifact_object.__repr__())[:self.max_preview_size_bytes]
            except Exception:
                artifact_type_data.preview = preview or ''
            delete_after_upload = True
            override_filename_ext_in_uri = '.pkl'
            override_filename_in_uri = name + override_filename_ext_in_uri
            fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
            os.close(fd)
            # noinspection PyBroadException
            try:
                with open(local_filename, 'wb') as f:
                    pickle.dump(artifact_object, f)
            except Exception:
                # cleanup and raise exception
                os.unlink(local_filename)
                raise

        # verify preview not out of scope:
        if artifact_type_data.preview and len(artifact_type_data.preview) > (self.max_preview_size_bytes+1024):
            artifact_type_data.preview = '# full preview too large to store, storing first {}kb\n{}'.format(
                self.max_preview_size_bytes // 1024, artifact_type_data.preview[:self.max_preview_size_bytes]
            )

        # remove from existing list, if exists
        for artifact in self._task_artifact_list:
            if artifact.key == name:
                if artifact.type == self._pd_artifact_type:
                    raise ValueError("Artifact of name {} already registered, "
                                     "use register_artifact instead".format(name))

                self._task_artifact_list.remove(artifact)
                break

        if not local_filename:
            file_size = None
            file_hash = None
        else:
            # check that the file to upload exists
            local_filename = Path(local_filename).absolute()
            if not local_filename.exists() or not local_filename.is_file():
                LoggerRoot.get_base_logger().warning('Artifact upload failed, cannot find file {}'.format(
                    local_filename.as_posix()))
                return False

            file_hash, _ = sha256sum(local_filename.as_posix(), block_size=Artifacts._hash_block_size)
            file_size = local_filename.stat().st_size

            uri = self._upload_local_file(local_filename, name,
                                          delete_after_upload=delete_after_upload,
                                          override_filename=override_filename_in_uri,
                                          override_filename_ext=override_filename_ext_in_uri,
                                          wait_on_upload=wait_on_upload)

        timestamp = int(time())

        artifact = tasks.Artifact(key=name, type=artifact_type,
                                  uri=uri,
                                  content_size=file_size,
                                  hash=file_hash,
                                  timestamp=timestamp,
                                  type_data=artifact_type_data,
                                  display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None)

        # update task artifacts
        self._add_artifact(artifact)

        return True

    def flush(self):
        # type: () -> ()
        # start the thread if it hasn't already:
        self._start()
        # flush the current state of all artifacts
        self._flush_event.set()

    def stop(self, wait=True):
        # type: (bool) -> ()
        # stop the daemon thread and quit
        # wait until thread exists
        self._exit_flag = True
        self._flush_event.set()
        if wait:
            if self._thread:
                self._thread.join()
            # remove all temp folders
            for f in self._temp_folder:
                # noinspection PyBroadException
                try:
                    Path(f).rmdir()
                except Exception:
                    pass

    def _start(self):
        # type: () -> ()
        """ Start daemon thread if any artifacts are registered and thread is not up yet """
        if not self._thread and self._artifacts_container:
            # start the daemon thread
            self._flush_event.clear()
            self._thread = Thread(target=self._daemon)
            self._thread.daemon = True
            self._thread.start()

    def _daemon(self):
        # type: () -> ()
        while not self._exit_flag:
            self._flush_event.wait(self._flush_frequency_sec)
            self._flush_event.clear()
            artifact_keys = list(self._artifacts_container.keys())
            for name in artifact_keys:
                try:
                    self._upload_data_audit_artifacts(name)
                except Exception as e:
                    LoggerRoot.get_base_logger().warning(str(e))

        # create summary
        self._summary = self._get_statistics()

    def _add_artifact(self, artifact):
        if not self._task:
            raise ValueError("Task object not set")
        with self._task_edit_lock:
            if artifact not in self._task_artifact_list:
                self._task_artifact_list.append(artifact)
            # noinspection PyProtectedMember
            self._task._add_artifacts(self._task_artifact_list)

    def _upload_data_audit_artifacts(self, name):
        # type: (str) -> ()
        logger = self._task.get_logger()
        pd_artifact = self._artifacts_container.get(name)
        pd_metadata = self._artifacts_container.get_metadata(name)

        # remove from artifacts watch list
        if name in self._unregister_request:
            try:
                self._unregister_request.remove(name)
            except KeyError:
                pass
            self._artifacts_container.unregister_artifact(name)

        if pd_artifact is None:
            return

        override_filename_ext_in_uri = self._save_format
        override_filename_in_uri = name
        fd, local_csv = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri)
        os.close(fd)
        local_csv = Path(local_csv)
        pd_artifact.to_csv(local_csv.as_posix(), index=False, compression=self._compression)
        current_sha2, file_sha2 = sha256sum(
            local_csv.as_posix(), skip_header=32, block_size=Artifacts._hash_block_size)
        if name in self._last_artifacts_upload:
            previous_sha2 = self._last_artifacts_upload[name]
            if previous_sha2 == current_sha2:
                # nothing to do, we can skip the upload
                # noinspection PyBroadException
                try:
                    local_csv.unlink()
                except Exception:
                    pass
                return
        self._last_artifacts_upload[name] = current_sha2

        # If old clearml-server, upload as debug image
        if not Session.check_min_api_version('2.3'):
            logger.report_image(title='artifacts', series=name, local_path=local_csv.as_posix(),
                                delete_after_upload=True, iteration=self._task.get_last_iteration(),
                                max_image_history=2)
            return

        # Find our artifact
        artifact = None
        for an_artifact in self._task_artifact_list:
            if an_artifact.key == name:
                artifact = an_artifact
                break

        file_size = local_csv.stat().st_size

        # upload file
        uri = self._upload_local_file(local_csv, name, delete_after_upload=True,
                                      override_filename=override_filename_in_uri,
                                      override_filename_ext=override_filename_ext_in_uri)

        # update task artifacts
        with self._task_edit_lock:
            if not artifact:
                artifact = tasks.Artifact(key=name, type=self._pd_artifact_type)
            artifact_type_data = tasks.ArtifactTypeData()

            artifact_type_data.data_hash = current_sha2
            artifact_type_data.content_type = "text/csv"
            artifact_type_data.preview = str(pd_artifact.__repr__(
            )) + '\n\n' + self._get_statistics({name: pd_artifact})

            artifact.type_data = artifact_type_data
            artifact.uri = uri
            artifact.content_size = file_size
            artifact.hash = file_sha2
            artifact.timestamp = int(time())
            artifact.display_data = [(str(k), str(v)) for k, v in pd_metadata.items()] if pd_metadata else None

            self._add_artifact(artifact)

    def _upload_local_file(
            self, local_file, name, delete_after_upload=False, override_filename=None, override_filename_ext=None,
            wait_on_upload=False
    ):
        # type: (str, str, bool, Optional[str], Optional[str], bool) -> str
        """
        Upload local file and return uri of the uploaded file (uploading in the background)
        """
        from clearml.storage import StorageManager

        upload_uri = self._task.output_uri or self._task.get_logger().get_default_upload_destination()
        if not isinstance(local_file, Path):
            local_file = Path(local_file)
        ev = UploadEvent(metric='artifacts', variant=name,
                         image_data=None, upload_uri=upload_uri,
                         local_image_path=local_file.as_posix(),
                         delete_after_upload=delete_after_upload,
                         override_filename=override_filename,
                         override_filename_ext=override_filename_ext,
                         override_storage_key_prefix=self._get_storage_uri_prefix())
        _, uri = ev.get_target_full_upload_uri(upload_uri, quote_uri=False)

        # send for upload
        # noinspection PyProtectedMember
        if wait_on_upload:
            StorageManager.upload_file(local_file.as_posix(), uri, wait_for_upload=True, retries=ev.retries)
            if delete_after_upload:
                try:
                    os.unlink(local_file.as_posix())
                except OSError:
                    LoggerRoot.get_base_logger().warning('Failed removing temporary {}'.format(local_file))
        else:
            self._task._reporter._report(ev)

        _, quoted_uri = ev.get_target_full_upload_uri(upload_uri)

        return quoted_uri

    def _get_statistics(self, artifacts_dict=None):
        # type: (Optional[Dict[str, Artifact]]) -> str
        summary = ''
        artifacts_dict = artifacts_dict or self._artifacts_container
        thread_pool = ThreadPool()

        try:
            # build hash row sets
            artifacts_summary = []
            for a_name, a_df in artifacts_dict.items():
                hash_cols = self._artifacts_container.get_hash_columns(a_name)
                if not pd or not isinstance(a_df, pd.DataFrame):
                    continue

                if hash_cols is True:
                    hash_col_drop = []
                else:
                    hash_cols = set(hash_cols)
                    missing_cols = hash_cols.difference(a_df.columns)
                    if missing_cols == hash_cols:
                        LoggerRoot.get_base_logger().warning(
                            'Uniqueness columns {} not found in artifact {}. '
                            'Skipping uniqueness check for artifact.'.format(list(missing_cols), a_name)
                        )
                        continue
                    elif missing_cols:
                        # missing_cols must be a subset of hash_cols
                        hash_cols.difference_update(missing_cols)
                        LoggerRoot.get_base_logger().warning(
                            'Uniqueness columns {} not found in artifact {}. Using {}.'.format(
                                list(missing_cols), a_name, list(hash_cols)
                            )
                        )

                    hash_col_drop = [col for col in a_df.columns if col not in hash_cols]

                a_unique_hash = set()

                def hash_row(r):
                    a_unique_hash.add(hash(bytes(r)))

                a_shape = a_df.shape
                # parallelize
                a_hash_cols = a_df.drop(columns=hash_col_drop)
                thread_pool.map(hash_row, a_hash_cols.values)
                # add result
                artifacts_summary.append((a_name, a_shape, a_unique_hash,))

            # build intersection summary
            for i, (name, shape, unique_hash) in enumerate(artifacts_summary):
                summary += '[{name}]: shape={shape}, {unique} unique rows, {percentage:.1f}% uniqueness\n'.format(
                    name=name, shape=shape, unique=len(unique_hash),
                    percentage=100 * len(unique_hash) / float(shape[0]))
                for name2, shape2, unique_hash2 in artifacts_summary[i + 1:]:
                    intersection = len(unique_hash & unique_hash2)
                    summary += '\tIntersection with [{name2}] {intersection} rows: {percentage:.1f}%\n'.format(
                        name2=name2, intersection=intersection,
                        percentage=100 * intersection / float(len(unique_hash2)))
        except Exception as e:
            LoggerRoot.get_base_logger().warning(str(e))
        finally:
            thread_pool.close()
            thread_pool.terminate()
        return summary

    def _get_temp_folder(self, force_new=False):
        # type: (bool) -> str
        if force_new or not self._temp_folder:
            new_temp = mkdtemp(prefix='artifacts_')
            self._temp_folder.append(new_temp)
            return new_temp
        return self._temp_folder[0]

    def _get_storage_uri_prefix(self):
        # type: () -> str
        if not self._storage_prefix:
            # noinspection PyProtectedMember
            self._storage_prefix = self._task._get_output_destination_suffix()
        return self._storage_prefix


# clearml/binding/click_bind.py

try:
    from click.core import Command, Option, Argument, Group, Context  # noqa
    from click.types import BoolParamType  # noqa
except ImportError:
    Command = None

from .frameworks import _patched_call  # noqa
from ..config import running_remotely, get_remote_task_id
from ..utilities.dicts import cast_str_to_bool


class PatchClick:
    _args = {}
    _args_desc = {}
    _args_type = {}
    _num_commands = 0
    _command_type = 'click.Command'
    _section_name = 'Args'
    _main_task = None
    __remote_task_params = None
    __remote_task_params_dict = {}
    __patched = False

    @classmethod
    def patch(cls, task=None):
        if Command is None:
            return

        if task:
            cls._main_task = task
            PatchClick._update_task_args()

        if not cls.__patched:
            cls.__patched = True
            Command.__init__ = _patched_call(Command.__init__, PatchClick._command_init)
            Command.parse_args = _patched_call(Command.parse_args, PatchClick._parse_args)
            Context.__init__ = _patched_call(Context.__init__, PatchClick._context_init)

    @classmethod
    def args(cls):
        # remove prefix and main command
        if cls._num_commands == 1:
            cmd = sorted(cls._args.keys())[0]
            skip = len(cmd)+1
        else:
            skip = 0

        _args = {cls._section_name+'/'+k[skip:]: v for k, v in cls._args.items() if k[skip:]}
        _args_type = {cls._section_name+'/'+k[skip:]: v for k, v in cls._args_type.items() if k[skip:]}
        _args_desc = {cls._section_name+'/'+k[skip:]: v for k, v in cls._args_desc.items() if k[skip:]}

        return _args, _args_type, _args_desc

    @classmethod
    def _update_task_args(cls):
        if running_remotely() or not cls._main_task or not cls._args:
            return
        param_val, param_types, param_desc = cls.args()
        # noinspection PyProtectedMember
        cls._main_task._set_parameters(
            param_val,
            __update=True,
            __parameters_descriptions=param_desc,
            __parameters_types=param_types
        )

    @staticmethod
    def _command_init(original_fn, self, *args, **kwargs):
        if self and isinstance(self, Command) and 'name' in kwargs:
            PatchClick._num_commands += 1
            if running_remotely():
                pass
            else:
                name = kwargs['name']
                if name:
                    PatchClick._args[name] = False
                    PatchClick._args_type[name] = PatchClick._command_type
                    # maybe we should take it post initialization
                    if kwargs.get('help'):
                        PatchClick._args_desc[name] = str(kwargs.get('help'))

                for option in kwargs.get('params') or []:
                    if not option or not isinstance(option, (Option, Argument)) or \
                            not getattr(option, 'expose_value', True):
                        continue
                    # store default value
                    PatchClick._args[name+'/'+option.name] = str(option.default or '')
                    # store value type
                    if option.type is not None:
                        PatchClick._args_type[name+'/'+option.name] = str(option.type)
                    # store value help
                    if getattr(option, 'help', None):
                        PatchClick._args_desc[name+'/'+option.name] = str(option.help)

        return original_fn(self, *args, **kwargs)

    @staticmethod
    def _parse_args(original_fn, self, *args, **kwargs):
        if running_remotely() and isinstance(self, Command) and isinstance(self, Group):
            command = PatchClick._load_task_params()
            if command:
                init_args = kwargs['args'] if 'args' in kwargs else args[1]
                init_args = [command] + (init_args[1:] if init_args else [])
                if 'args' in kwargs:
                    kwargs['args'] = init_args
                else:
                    args = (args[0], init_args) + args[2:]

        ret = original_fn(self, *args, **kwargs)

        if isinstance(self, Command) and not isinstance(self, Group):
            ctx = kwargs.get('ctx') or args[0]
            if running_remotely():
                PatchClick._load_task_params()
                for p in self.params:
                    name = '{}/{}'.format(self.name, p.name) if PatchClick._num_commands > 1 else p.name
                    value = PatchClick.__remote_task_params_dict.get(name)
                    ctx.params[p.name] = p.process_value(
                        ctx, cast_str_to_bool(value, strip=True) if isinstance(p.type, BoolParamType) else value)
            else:
                PatchClick._args[self.name] = True
                for k, v in ctx.params.items():
                    # store passed value
                    PatchClick._args[self.name + '/' + str(k)] = str(v or '')

                PatchClick._update_task_args()
        return ret

    @staticmethod
    def _context_init(original_fn, self, *args, **kwargs):
        if running_remotely():
            kwargs['resilient_parsing'] = True
        return original_fn(self, *args, **kwargs)

    @staticmethod
    def _load_task_params():
        if not PatchClick.__remote_task_params:
            from clearml import Task
            t = Task.get_task(task_id=get_remote_task_id())
            # noinspection PyProtectedMember
            PatchClick.__remote_task_params = t._get_task_property('hyperparams') or {}
            params_dict = t.get_parameters(backwards_compatibility=False)
            skip = len(PatchClick._section_name)+1
            PatchClick.__remote_task_params_dict = {
                k[skip:]: v for k, v in params_dict.items()
                if k.startswith(PatchClick._section_name+'/')
            }

        params = PatchClick.__remote_task_params
        command = [
            p.name for p in params['Args'].values()
            if p.type == PatchClick._command_type and cast_str_to_bool(p.value, strip=True)]
        return command[0] if command else None


# patch click before anything
PatchClick.patch()


# clearml/binding/environ_bind.py

import os
from time import sleep

import six

from ..config import TASK_LOG_ENVIRONMENT, running_remotely, config
from ..utilities.process.mp import BackgroundMonitor


class EnvironmentBind(object):
    _task = None
    _environment_section = 'Environment'

    @classmethod
    def update_current_task(cls, current_task):
        cls._task = current_task
        # noinspection PyBroadException
        try:
            cls._bind_environment()
        except Exception:
            pass

    @classmethod
    def _bind_environment(cls):
        if not cls._task:
            return

        # get ENVIRONMENT and put it into the OS environment
        if running_remotely():
            params = cls._task.get_parameters_as_dict()
            if params and cls._environment_section in params:
                # put back into os:
                os.environ.update(params[cls._environment_section])
            return

        environ_log = \
            str(TASK_LOG_ENVIRONMENT.get() or '').strip() or config.get('development.log_os_environments', [])
        if environ_log and isinstance(environ_log, str):
            environ_log = [e.strip() for e in environ_log.split(',')]

        if not environ_log:
            return

        env_param = dict()
        for match in environ_log:
            match = match.strip()
            if match == '*':
                env_param.update({k: os.environ.get(k) for k in os.environ
                                  if not k.startswith('TRAINS_') and not k.startswith('CLEARML_')})
            elif match.endswith('*'):
                match = match.rstrip('*')
                env_param.update({k: os.environ.get(k) for k in os.environ if k.startswith(match)})
            elif match.startswith('*'):
                match = match.lstrip('*')
                env_param.update({k: os.environ.get(k) for k in os.environ if k.endswith(match)})
            elif match in os.environ:
                env_param.update({match: os.environ.get(match)})
        # store os environments
        cls._task.connect(env_param, cls._environment_section)


class PatchOsFork(object):
    _original_fork = None

    @classmethod
    def patch_fork(cls):
        # noinspection PyBroadException
        try:
            # only once
            if cls._original_fork:
                return
            if six.PY2:
                cls._original_fork = staticmethod(os.fork)
            else:
                cls._original_fork = os.fork
            os.fork = cls._patched_fork
        except Exception:
            pass

    @staticmethod
    def _patched_fork(*args, **kwargs):
        ret = PatchOsFork._original_fork(*args, **kwargs)
        # Make sure the new process stdout is logged
        if not ret:
            from ..task import Task
            # force creating a Task
            task = Task.current_task()
            # # Hack: now make sure we setup the reporter threads (Log+Reporter)
            if not task._report_subprocess_enabled:
                BackgroundMonitor.start_all(task=task)

            # The signal handler method is Not enough, for the time being, we have both
            # even though it makes little sense
            # # if we got here patch the os._exit of our instance to call us
            def _at_exit_callback(*a_args, **a_kwargs):
                # just make sure we flush the internal state (the at exist caught by the external signal does the rest
                # in theory we should not have to do any of that, but for some reason if we do not
                # the signal is never caught by the signal call backs, not sure why....
                sleep(0.1)
                # noinspection PyProtectedMember, PyUnresolvedReferences
                return os._org_exit(*a_args, **a_kwargs)

            if not hasattr(os, '_org_exit'):
                # noinspection PyProtectedMember, PyUnresolvedReferences
                os._org_exit = os._exit

            os._exit = _at_exit_callback

        return ret


# clearml/binding/hydra_bind.py

import io
import sys
from functools import partial

from ..config import running_remotely, get_remote_task_id, DEV_TASK_NO_REUSE
from ..debugging.log import LoggerRoot


class PatchHydra(object):
    _original_run_job = None
    _original_hydra_run = None
    _allow_omegaconf_edit = None
    _current_task = None
    _last_untracked_state = {}
    _config_section = 'OmegaConf'
    _parameter_section = 'Hydra'
    _parameter_allow_full_edit = '_allow_omegaconf_edit_'

    @classmethod
    def patch_hydra(cls):
        # noinspection PyBroadException
        try:
            # only once
            if cls._original_run_job:
                return True
            # if hydra is not loaded, do not patch anything
            if not sys.modules.get('hydra'):
                return False

            from hydra._internal import hydra as internal_hydra  # noqa
            from hydra.core import utils as utils_hydra  # noqa
            from hydra._internal.core_plugins import basic_launcher  # noqa

            cls._original_hydra_run = internal_hydra.Hydra.run
            internal_hydra.Hydra.run = cls._patched_hydra_run

            cls._original_run_job = utils_hydra.run_job
            utils_hydra.run_job = cls._patched_run_job
            internal_hydra.run_job = cls._patched_run_job
            basic_launcher.run_job = cls._patched_run_job
            return True
        except Exception:
            return False

    @staticmethod
    def update_current_task(task):
        # set current Task before patching
        PatchHydra._current_task = task
        if PatchHydra.patch_hydra():
            # check if we have an untracked state, store it.
            if PatchHydra._last_untracked_state.get('connect'):
                PatchHydra._current_task.connect(**PatchHydra._last_untracked_state['connect'])
            if PatchHydra._last_untracked_state.get('_set_configuration'):
                # noinspection PyProtectedMember
                PatchHydra._current_task._set_configuration(**PatchHydra._last_untracked_state['_set_configuration'])
            PatchHydra._last_untracked_state = {}
        else:
            # if patching failed set it to None
            PatchHydra._current_task = None

    @staticmethod
    def _patched_hydra_run(self, config_name, task_function, overrides, *args, **kwargs):
        PatchHydra._allow_omegaconf_edit = False

        # store the config
        # noinspection PyBroadException
        try:
            if running_remotely():
                if not PatchHydra._current_task:
                    from ..task import Task
                    PatchHydra._current_task = Task.get_task(task_id=get_remote_task_id())
                # get the _parameter_allow_full_edit casted back to boolean
                connected_config = dict()
                connected_config[PatchHydra._parameter_allow_full_edit] = False
                PatchHydra._current_task.connect(connected_config, name=PatchHydra._parameter_section)
                PatchHydra._allow_omegaconf_edit = connected_config.pop(PatchHydra._parameter_allow_full_edit, None)
                # get all the overrides
                full_parameters = PatchHydra._current_task.get_parameters(backwards_compatibility=False)
                stored_config = {k[len(PatchHydra._parameter_section)+1:]: v for k, v in full_parameters.items()
                                 if k.startswith(PatchHydra._parameter_section+'/')}
                stored_config.pop(PatchHydra._parameter_allow_full_edit, None)
                overrides = ['{}={}'.format(k, v) for k, v in stored_config.items()]
            else:
                # We take care of it inside the _patched_run_job
                pass
        except Exception:
            pass

        return PatchHydra._original_hydra_run(self, config_name, task_function, overrides, *args, **kwargs)

    @staticmethod
    def _patched_run_job(config, task_function, *args, **kwargs):
        # store the config
        # noinspection PyBroadException
        try:
            if running_remotely():
                # we take care of it in the hydra run (where we have access to the overrides)
                pass
            else:
                overrides = config.hydra.overrides.task
                stored_config = dict(arg.split('=', 1) for arg in overrides)
                stored_config[PatchHydra._parameter_allow_full_edit] = False
                if PatchHydra._current_task:
                    PatchHydra._current_task.connect(stored_config, name=PatchHydra._parameter_section)
                    PatchHydra._last_untracked_state.pop('connect', None)
                else:
                    PatchHydra._last_untracked_state['connect'] = dict(
                        mutable=stored_config, name=PatchHydra._parameter_section)
                # todo: remove the overrides section from the Args (we have it here)
                # PatchHydra._current_task.delete_parameter('Args/overrides')
        except Exception:
            pass

        pre_app_task_init_call = bool(PatchHydra._current_task)

        if pre_app_task_init_call and not running_remotely():
            LoggerRoot.get_base_logger(PatchHydra).info(
                'Task.init called outside of Hydra-App. For full Hydra multi-run support, '
                'move the Task.init call into the Hydra-App main function')

        kwargs["config"] = config
        kwargs["task_function"] = partial(PatchHydra._patched_task_function, task_function,)
        result = PatchHydra._original_run_job(*args, **kwargs)

        # if we have Task.init called inside the App, we close it after the app is done.
        # This will make sure that hydra run will create multiple Tasks
        if not running_remotely() and not pre_app_task_init_call and PatchHydra._current_task:
            PatchHydra._current_task.close()
            # make sure we do not reuse the Task if we have a multi-run session
            DEV_TASK_NO_REUSE.set(True)
            PatchHydra._current_task = None

        return result

    @staticmethod
    def _patched_task_function(task_function, a_config, *a_args, **a_kwargs):
        from omegaconf import OmegaConf  # noqa
        if not running_remotely() or not PatchHydra._allow_omegaconf_edit:
            PatchHydra._register_omegaconf(a_config)
        else:
            # noinspection PyProtectedMember
            omega_yaml = PatchHydra._current_task._get_configuration_text(PatchHydra._config_section)
            loaded_config = OmegaConf.load(io.StringIO(omega_yaml))
            a_config = OmegaConf.merge(a_config, loaded_config)
            PatchHydra._register_omegaconf(a_config, is_read_only=False)
        return task_function(a_config, *a_args, **a_kwargs)

    @staticmethod
    def _register_omegaconf(config, is_read_only=True):
        from omegaconf import OmegaConf  # noqa

        if is_read_only:
            description = \
                'Full OmegaConf YAML configuration. ' \
                'This is a read-only section, unless \'{}/{}\' is set to True'.format(
                    PatchHydra._parameter_section, PatchHydra._parameter_allow_full_edit)
        else:
            description = 'Full OmegaConf YAML configuration overridden! ({}/{}=True)'.format(
                PatchHydra._parameter_section, PatchHydra._parameter_allow_full_edit)

        # we should not have the hydra section in the config, but this seems never to be the case anymore.
        # config = config.copy()
        # config.pop('hydra', None)

        configuration = dict(
            name=PatchHydra._config_section,
            description=description,
            config_type='OmegaConf YAML',
            config_text=OmegaConf.to_yaml(config, resolve=False)
        )
        if PatchHydra._current_task:
            # noinspection PyProtectedMember
            PatchHydra._current_task._set_configuration(**configuration)
            PatchHydra._last_untracked_state.pop('_set_configuration', None)
        else:
            PatchHydra._last_untracked_state['_set_configuration'] = configuration


def __global_hydra_bind():
    # noinspection PyBroadException
    try:
        import hydra  # noqa
        PatchHydra.patch_hydra()
    except Exception:
        pass


# patch hydra
__global_hydra_bind()


# clearml/binding/import_bind.py

import sys
from collections import defaultdict
import six

if six.PY2:
    # python2.x
    import __builtin__ as builtins
else:
    # python3.x
    import builtins


class PostImportHookPatching(object):
    _patched = False
    _post_import_hooks = defaultdict(list)

    @staticmethod
    def _init_hook():
        if PostImportHookPatching._patched:
            return
        PostImportHookPatching._patched = True

        if six.PY2:
            # python2.x
            builtins.__org_import__ = builtins.__import__
            builtins.__import__ = PostImportHookPatching.__patched_import2
        else:
            # python3.x
            builtins.__org_import__ = builtins.__import__
            builtins.__import__ = PostImportHookPatching.__patched_import3

    @staticmethod
    def __patched_import2(name, globals={}, locals={}, fromlist=[], level=-1):
        already_imported = name in sys.modules
        mod = builtins.__org_import__(
            name,
            globals=globals,
            locals=locals,
            fromlist=fromlist,
            level=level)

        if not already_imported and name in PostImportHookPatching._post_import_hooks:
            for hook in PostImportHookPatching._post_import_hooks[name]:
                hook()
        return mod

    @staticmethod
    def __patched_import3(name, globals=None, locals=None, fromlist=(), level=0):
        name_parts = name.split('.')
        base_name = name_parts[0]
        second_name = '.'.join(name_parts[:2]) if len(name_parts) > 1 else None
        base_already_imported = (not base_name) or (base_name in sys.modules)
        second_already_imported = (not second_name) or (second_name in sys.modules)
        mod = builtins.__org_import__(
            name,
            globals=globals,
            locals=locals,
            fromlist=fromlist,
            level=level)
        if not base_already_imported and base_name in PostImportHookPatching._post_import_hooks:
            for hook in PostImportHookPatching._post_import_hooks[base_name]:
                hook()
        if not second_already_imported and second_name in PostImportHookPatching._post_import_hooks:
            for hook in PostImportHookPatching._post_import_hooks[second_name]:
                hook()
        return mod

    @staticmethod
    def add_on_import(name, func):
        PostImportHookPatching._init_hook()
        if name not in PostImportHookPatching._post_import_hooks or \
                func not in PostImportHookPatching._post_import_hooks[name]:
            PostImportHookPatching._post_import_hooks[name].append(func)

    @staticmethod
    def remove_on_import(name, func):
        if name in PostImportHookPatching._post_import_hooks and \
                func in PostImportHookPatching._post_import_hooks[name]:
            PostImportHookPatching._post_import_hooks[name].remove(func)


# clearml/binding/joblib_bind.py

import sys
import warnings
from functools import partial

import six
from pathlib2 import Path

from .import_bind import PostImportHookPatching
from ..binding.frameworks import _patched_call, _Empty, WeightsFileHandler
from ..config import running_remotely
from ..debugging.log import LoggerRoot
from ..model import Framework
from ..utilities.lowlevel.file_access import get_filename_from_file_object, buffer_writer_close_cb


class PatchedJoblib(object):
    _patched_joblib = False
    _patched_sk_joblib = False
    _current_task = None

    @staticmethod
    def patch_joblib():
        # try manually
        PatchedJoblib._patch_joblib()
        # register callback
        PostImportHookPatching.add_on_import('joblib',
                                             PatchedJoblib._patch_joblib)
        PostImportHookPatching.add_on_import('sklearn',
                                             PatchedJoblib._patch_joblib)

    @staticmethod
    def _patch_joblib():
        # noinspection PyBroadException
        try:
            if not PatchedJoblib._patched_joblib and 'joblib' in sys.modules:
                PatchedJoblib._patched_joblib = True
                try:
                    import joblib
                except ImportError:
                    joblib = None

                if joblib:
                    joblib.numpy_pickle._write_fileobject = _patched_call(
                        joblib.numpy_pickle._write_fileobject,
                        partial(PatchedJoblib._write_fileobject, joblib.numpy_pickle))
                    joblib.numpy_pickle._read_fileobject = _patched_call(
                        joblib.numpy_pickle._read_fileobject, PatchedJoblib._load)
                    joblib.numpy_pickle.NumpyPickler.__init__ = _patched_call(
                        joblib.numpy_pickle.NumpyPickler.__init__,
                        PatchedJoblib._numpypickler)

            if not PatchedJoblib._patched_sk_joblib and 'sklearn' in sys.modules:
                PatchedJoblib._patched_sk_joblib = True
                try:
                    import sklearn  # noqa: F401
                    # avoid deprecation warning, we must import sklearn before, so we could catch it
                    with warnings.catch_warnings():
                        warnings.simplefilter("ignore")
                        from sklearn.externals import joblib as sk_joblib
                except ImportError:
                    sk_joblib = None

                if sk_joblib:
                    sk_joblib.numpy_pickle._write_fileobject = _patched_call(
                        sk_joblib.numpy_pickle._write_fileobject,
                        partial(PatchedJoblib._write_fileobject, sk_joblib.numpy_pickle))
                    sk_joblib.numpy_pickle._read_fileobject = _patched_call(
                        sk_joblib.numpy_pickle._read_fileobject, PatchedJoblib._load)
                    sk_joblib.numpy_pickle.NumpyPickler.__init__ = _patched_call(
                        sk_joblib.numpy_pickle.NumpyPickler.__init__,
                        PatchedJoblib._numpypickler)

        except Exception:
            return False
        return True

    @staticmethod
    def update_current_task(task):
        PatchedJoblib._current_task = task
        PatchedJoblib.patch_joblib()

    @staticmethod
    def _dump(original_fn, obj, f, *args, **kwargs):
        ret = original_fn(obj, f, *args, **kwargs)
        if not PatchedJoblib._current_task:
            return ret
        PatchedJoblib._register_dump(obj, f)
        return ret

    @staticmethod
    def _numpypickler(original_fn, obj, f, *args, **kwargs):
        ret = original_fn(obj, f, *args, **kwargs)
        if not PatchedJoblib._current_task:
            return ret

        fname = f if isinstance(f, six.string_types) else None
        fileobj = ret if isinstance(f, six.string_types) else f

        if fileobj and hasattr(fileobj, 'close'):
            def callback(*_):
                PatchedJoblib._register_dump(obj, fname or fileobj)

            if isinstance(fname, six.string_types) or hasattr(fileobj, 'name'):
                buffer_writer_close_cb(fileobj, callback)
        else:
            PatchedJoblib._register_dump(obj, f)

        return ret

    @staticmethod
    def _write_fileobject(obj, original_fn, f, *args, **kwargs):
        ret = original_fn(f, *args, **kwargs)
        if not PatchedJoblib._current_task:
            return ret

        fname = f if isinstance(f, six.string_types) else None
        fileobj = ret if isinstance(f, six.string_types) else f

        if fileobj and hasattr(fileobj, 'close'):
            def callback(*_):
                PatchedJoblib._register_dump(obj, fname or fileobj)

            if isinstance(fname, six.string_types) or hasattr(fileobj, 'name'):
                buffer_writer_close_cb(fileobj, callback)
        else:
            PatchedJoblib._register_dump(obj, f)
        return ret

    @staticmethod
    def _register_dump(obj, f):
        filename = get_filename_from_file_object(f, flush=True)
        if not filename:
            return

        # give the model a descriptive name based on the file name
        # noinspection PyBroadException
        try:
            model_name = Path(filename).stem
        except Exception:
            model_name = None
        current_framework = PatchedJoblib.get_model_framework(obj)
        WeightsFileHandler.create_output_model(obj, filename, current_framework,
                                               PatchedJoblib._current_task, singlefile=True, model_name=model_name)

    @staticmethod
    def _load(original_fn, f, *args, **kwargs):
        if not PatchedJoblib._current_task:
            return original_fn(f, *args, **kwargs)

        filename = get_filename_from_file_object(f, flush=False)

        # register input model
        empty = _Empty()
        # Hack: disabled
        if False and running_remotely():
            # we assume scikit-learn, for the time being
            current_framework = Framework.scikitlearn
            filename = WeightsFileHandler.restore_weights_file(empty, filename, current_framework,
                                                               PatchedJoblib._current_task)
            model = original_fn(filename or f, *args, **kwargs)
        else:
            # try to load model before registering, in case we fail
            model = original_fn(f, *args, **kwargs)
            current_framework = PatchedJoblib.get_model_framework(model)
            WeightsFileHandler.restore_weights_file(empty, filename, current_framework,
                                                    PatchedJoblib._current_task)

        if empty.trains_in_model:
            # noinspection PyBroadException
            try:
                model.trains_in_model = empty.trains_in_model
            except Exception:
                pass
        return model

    @staticmethod
    def get_model_framework(obj):
        framework = Framework.scikitlearn
        object_orig_module = None
        # noinspection PyBroadException
        try:
            object_orig_module = obj.__module__ if hasattr(obj, '__module__') else obj.__package__
            model = object_orig_module.partition(".")[0]
            if model == 'sklearn':
                framework = Framework.scikitlearn
            elif model == 'xgboost':
                framework = Framework.xgboost
            else:
                framework = Framework.scikitlearn
        except Exception:
            LoggerRoot.get_base_logger().debug(
                "Can't get model framework {}, model framework will be: {} ".format(object_orig_module, framework))
        finally:
            return framework


# clearml/binding/jsonargs_bind.py

""" jsonargparse binding utility functions """
from ..config import running_remotely


class PatchJsonArgParse(object):
    _original_parse_call = None
    _task = None

    @classmethod
    def update_current_task(cls, current_task):
        cls._task = current_task
        cls._patch_jsonargparse()

    @classmethod
    def _patch_jsonargparse(cls):
        # already patched
        if cls._original_parse_call:
            return

        # noinspection PyBroadException
        try:
            from jsonargparse import ArgumentParser  # noqa
            cls._original_parse_call = ArgumentParser._parse_common  # noqa
            ArgumentParser._parse_common = cls._patched_parse_known_args
        except Exception:
            # there is no jsonargparse
            pass

    @staticmethod
    def _patched_parse_known_args(self, *args, **kwargs):
        if not PatchJsonArgParse._task:
            return PatchJsonArgParse._original_parse_call(self, *args, **kwargs)

        try:
            from argparse import SUPPRESS
            from jsonargparse.typehints import ActionTypeHint
            from jsonargparse.actions import ActionConfigFile, _ActionSubCommands, \
                _ActionConfigLoad, filter_default_actions  # noqa
            from jsonargparse.util import get_key_value_from_flat_dict, update_key_value_in_flat_dict, \
                namespace_to_dict, _dict_to_flat_namespace  # noqa
        except ImportError:
            # something happened, let's just call the original
            return PatchJsonArgParse._original_parse_call(self, *args, **kwargs)

        def cleanup_actions(cfg, actions, prefix='', skip_none=False, cast_value=False):
            for action in filter_default_actions(actions):
                action_dest = prefix + action.dest
                if (action.help == SUPPRESS and not isinstance(action, _ActionConfigLoad)) or \
                   isinstance(action, ActionConfigFile) or \
                   (skip_none and action_dest in cfg and cfg[action_dest] is None):
                    cfg.pop(action_dest, None)
                elif isinstance(action, _ActionSubCommands):
                    for key, subparser in action.choices.items():
                        cleanup_actions(cfg, subparser._actions, prefix=prefix+key+'.',
                                        skip_none=skip_none, cast_value=cast_value)
                elif cast_value and isinstance(action, ActionTypeHint):
                    value = get_key_value_from_flat_dict(cfg, action_dest)
                    if value is not None and value != {}:
                        if value:
                            parsed_value = action._check_type(value)
                        else:
                            try:
                                parsed_value = action._check_type(None)
                            except TypeError:
                                # try with original empty text
                                parsed_value = action._check_type(value)

                        update_key_value_in_flat_dict(cfg, action_dest, parsed_value)
                elif cast_value and hasattr(action, 'type') and not isinstance(action, _ActionConfigLoad):
                    value = get_key_value_from_flat_dict(cfg, action_dest)
                    try:
                        parsed_value = action.type(value or None) if action.type != str else str(value)
                        update_key_value_in_flat_dict(cfg, action_dest, parsed_value)
                    except (ValueError, TypeError):
                        pass

        if not running_remotely():
            ret = PatchJsonArgParse._original_parse_call(self, *args, **kwargs)

            # noinspection PyBroadException
            try:
                cfg_dict = ret if isinstance(ret, dict) else namespace_to_dict(ret)
                cfg_dict = namespace_to_dict(_dict_to_flat_namespace(cfg_dict))
                cleanup_actions(cfg_dict, actions=self._actions, skip_none=False, cast_value=False)
            except Exception:
                cfg_dict = None

            # store / sync arguments
            if cfg_dict is not None:
                PatchJsonArgParse._task.connect(cfg_dict, name='Args')
        else:
            cfg_dict = PatchJsonArgParse._task.get_parameters_as_dict().get('Args', None)
            if cfg_dict is not None:
                if 'cfg' in kwargs:
                    cleanup_actions(cfg_dict, actions=self._actions, skip_none=False, cast_value=True)
                    kwargs['cfg'].update(cfg_dict)
                else:
                    print('Warning failed applying jsonargparse configuration')

            ret = PatchJsonArgParse._original_parse_call(self, *args, **kwargs)

        return ret


# clearml/binding/matplotlib_bind.py

# -*- coding: utf-8 -*-

import os
import sys
import threading
from copy import deepcopy
from tempfile import mkstemp

import six
from six import BytesIO

from .import_bind import PostImportHookPatching
from ..config import running_remotely
from ..debugging.log import LoggerRoot
from ..utilities.resource_monitor import ResourceMonitor


class PatchedMatplotlib:
    _patched_original_plot = None
    _patched_original_figure = None
    _patched_original_savefig = None
    _patched_original_imshow = None
    __patched_original_draw_all = None
    __patched_draw_all_recursion_guard = False
    _global_plot_counter = -1
    _global_image_counter = -1
    _global_image_counter_limit = None
    _last_iteration_plot_titles = {}
    _current_task = None
    _support_image_plot = None
    _matplotlylib = None
    _plotly_renderer = None
    _lock_renderer = threading.RLock()
    _recursion_guard = {}
    _matplot_major_version = 0
    _matplot_minor_version = 0
    _force_report_as_image = False
    _logger_started_reporting = False
    _matplotlib_reported_titles = set()

    class _PatchWarnings(object):
        def __init__(self):
            pass

        def warn(self, text, *args, **kwargs):
            raise ValueError(text)

        def __getattr__(self, item):
            def bypass(*_, **__):
                pass
            return bypass

    @staticmethod
    def force_report_as_image(force):
        # type: (bool) -> None
        """
        Set force_report_as_image. If True all matplotlib are always converted to images
        Otherwise we try to convert them into interactive plotly plots.
        :param force: True force
        """
        PatchedMatplotlib._force_report_as_image = bool(force)

    @staticmethod
    def patch_matplotlib():
        # only once
        if PatchedMatplotlib._patched_original_plot is not None:
            return True
        # make sure we only patch once
        PatchedMatplotlib._patched_original_plot = False

        # noinspection PyBroadException
        try:
            # we support matplotlib version 2.0.0 and above
            import matplotlib
            PatchedMatplotlib._update_matplotlib_version()
            if PatchedMatplotlib._matplot_major_version < 2:
                LoggerRoot.get_base_logger().warning(
                    'matplotlib binding supports version 2.0 and above, found version {}'.format(
                        matplotlib.__version__))
                PatchedMatplotlib._patched_original_plot = False
                return False

            if running_remotely():
                # disable GUI backend - make headless
                matplotlib.rcParams['backend'] = 'agg'
                import matplotlib.pyplot
                matplotlib.pyplot.switch_backend('agg')
            import matplotlib.pyplot as plt
            import matplotlib.figure as figure
            if six.PY2:
                PatchedMatplotlib._patched_original_plot = staticmethod(plt.show)
                PatchedMatplotlib._patched_original_imshow = staticmethod(plt.imshow)
                PatchedMatplotlib._patched_original_figure = staticmethod(figure.Figure.show)
                PatchedMatplotlib._patched_original_savefig = staticmethod(figure.Figure.savefig)
            else:
                PatchedMatplotlib._patched_original_plot = plt.show
                PatchedMatplotlib._patched_original_imshow = plt.imshow
                PatchedMatplotlib._patched_original_figure = figure.Figure.show
                PatchedMatplotlib._patched_original_savefig = figure.Figure.savefig

            # noinspection PyBroadException
            try:
                import matplotlib.pylab as pltlab
                if plt.show == pltlab.show:
                    pltlab.show = PatchedMatplotlib.patched_show
                if plt.imshow == pltlab.imshow:
                    pltlab.imshow = PatchedMatplotlib.patched_imshow
            except Exception:
                pass
            plt.show = PatchedMatplotlib.patched_show
            figure.Figure.show = PatchedMatplotlib.patched_figure_show
            sys.modules['matplotlib'].pyplot.imshow = PatchedMatplotlib.patched_imshow
            sys.modules['matplotlib'].figure.Figure.savefig = PatchedMatplotlib.patched_savefig
            # patch plotly so we know it failed us.
            from ..utilities.plotlympl import renderer
            # noinspection PyProtectedMember
            renderer.warnings = PatchedMatplotlib._PatchWarnings()

        except Exception:
            return False

        # patch IPython matplotlib inline mode
        # noinspection PyBroadException
        try:
            if 'IPython' in sys.modules:
                from IPython import get_ipython
                ip = get_ipython()
                if ip and matplotlib.is_interactive():
                    # instead of hooking ipython, we should hook the matplotlib
                    import matplotlib.pyplot as plt
                    PatchedMatplotlib.__patched_original_draw_all = plt.draw_all
                    plt.draw_all = PatchedMatplotlib.__patched_draw_all
                    # ip.events.register('post_execute', PatchedMatplotlib.ipython_post_execute_hook)
        except Exception:
            pass

        # update api version
        PatchedMatplotlib._update_matplotlib_image_support()

        # load plotly
        PatchedMatplotlib._update_plotly_renderers()

        return True

    @staticmethod
    def _update_matplotlib_image_support():
        if PatchedMatplotlib._support_image_plot is None:
            from ..backend_api import Session
            PatchedMatplotlib._support_image_plot = Session.check_min_api_version('2.2')

    @staticmethod
    def _update_matplotlib_version():
        if PatchedMatplotlib._matplot_major_version:
            return

        # we support matplotlib version 2.0.0 and above
        try:
            import matplotlib
            version_split = matplotlib.__version__.split('.')
            PatchedMatplotlib._matplot_major_version = int(version_split[0])
            PatchedMatplotlib._matplot_minor_version = int(version_split[1])

            if running_remotely():
                # disable GUI backend - make headless
                matplotlib.rcParams['backend'] = 'agg'
                import matplotlib.pyplot
                matplotlib.pyplot.switch_backend('agg')

        except Exception:
            pass

        # check backend support
        PatchedMatplotlib._update_matplotlib_image_support()

    @staticmethod
    def _update_plotly_renderers():
        if PatchedMatplotlib._matplotlylib and PatchedMatplotlib._plotly_renderer:
            return True

        # create plotly renderer
        # noinspection PyBroadException
        try:
            from ..utilities import plotlympl
            PatchedMatplotlib._matplotlylib = plotlympl
            PatchedMatplotlib._plotly_renderer = plotlympl.PlotlyRenderer()
        except Exception:
            return False

        return True

    @staticmethod
    def update_current_task(task):
        # make sure we have a default vale
        if PatchedMatplotlib._global_image_counter_limit is None:
            from ..config import config
            PatchedMatplotlib._global_image_counter_limit = config.get('metric.matplotlib_untitled_history_size', 100)

        # if we already patched it, just update the current task
        if PatchedMatplotlib._patched_original_plot is not None:
            PatchedMatplotlib._current_task = task
        # if matplotlib is not loaded yet, get a callback hook
        elif not running_remotely() and \
                ('matplotlib.pyplot' not in sys.modules and 'matplotlib.pylab' not in sys.modules):
            PatchedMatplotlib._current_task = task
            PostImportHookPatching.add_on_import('matplotlib.pyplot', PatchedMatplotlib.patch_matplotlib)
            PostImportHookPatching.add_on_import('matplotlib.pylab', PatchedMatplotlib.patch_matplotlib)
        elif PatchedMatplotlib.patch_matplotlib():
            PatchedMatplotlib._current_task = task

    @staticmethod
    def patched_imshow(*args, **kw):
        ret = PatchedMatplotlib._patched_original_imshow(*args, **kw)
        try:
            from matplotlib import _pylab_helpers
            # store on the plot that this is an imshow plot
            stored_figure = _pylab_helpers.Gcf.get_active()
            if stored_figure:
                stored_figure._trains_is_imshow = 1 if getattr(stored_figure, '_trains_is_imshow', None) is None \
                    else stored_figure._trains_is_imshow + 1
        except Exception:
            pass
        return ret

    @staticmethod
    def patched_savefig(self, *args, **kw):
        ret = PatchedMatplotlib._patched_original_savefig(self, *args, **kw)
        # noinspection PyBroadException
        try:
            fname = kw.get('fname') or args[0]
            from pathlib2 import Path
            if six.PY3:
                from pathlib import Path as Path3
            else:
                Path3 = Path

            # if we are not storing into a file (str/Path) do not log the matplotlib
            if not isinstance(fname, (str, Path, Path3)):
                return ret
        except Exception:
            pass

        tid = threading._get_ident() if six.PY2 else threading.get_ident()
        if not PatchedMatplotlib._recursion_guard.get(tid):
            PatchedMatplotlib._recursion_guard[tid] = True
            # noinspection PyBroadException
            try:
                PatchedMatplotlib._report_figure(specific_fig=self, set_active=False)
            except Exception:
                pass
            PatchedMatplotlib._recursion_guard[tid] = False

        return ret

    @staticmethod
    def patched_figure_show(self, *args, **kw):
        tid = threading._get_ident() if six.PY2 else threading.get_ident()
        if PatchedMatplotlib._recursion_guard.get(tid):
            # we are inside a gaurd do nothing
            return PatchedMatplotlib._patched_original_figure(self, *args, **kw)

        PatchedMatplotlib._recursion_guard[tid] = True
        PatchedMatplotlib._report_figure(set_active=False, specific_fig=self)
        ret = PatchedMatplotlib._patched_original_figure(self, *args, **kw)
        PatchedMatplotlib._recursion_guard[tid] = False
        return ret

    @staticmethod
    def patched_show(*args, **kw):
        tid = threading._get_ident() if six.PY2 else threading.get_ident()
        PatchedMatplotlib._recursion_guard[tid] = True
        # noinspection PyBroadException
        try:
            figures = PatchedMatplotlib._get_output_figures(None, all_figures=True)
            for figure in figures:
                # if this is a stale figure (just updated) we should send it, the rest will not be stale
                if figure.canvas.figure.stale or (
                        getattr(figure, '_trains_is_imshow', None) is not None and figure._trains_is_imshow):
                    PatchedMatplotlib._report_figure(stored_figure=figure)
        except Exception:
            pass
        ret = PatchedMatplotlib._patched_original_plot(*args, **kw)
        if PatchedMatplotlib._current_task and sys.modules['matplotlib'].rcParams['backend'] == 'agg':
            # clear the current plot, because no one else will
            # noinspection PyBroadException
            try:
                if sys.modules['matplotlib'].rcParams['backend'] == 'agg':
                    # noinspection PyBroadException
                    try:
                        from matplotlib import _pylab_helpers
                        stored_figure = _pylab_helpers.Gcf.get_active()
                        stored_figure._trains_is_imshow = None
                        stored_figure.canvas.figure._trains_explicit = False
                    except Exception:
                        pass
                    import matplotlib.pyplot as plt
                    plt.clf()
            except Exception:
                pass
        PatchedMatplotlib._recursion_guard[tid] = False
        return ret

    @staticmethod
    def report_figure(title, series, figure, iter,
                      force_save_as_image=False, report_as_debug_sample=False, reporter=None, logger=None):
        PatchedMatplotlib._report_figure(
            force_save_as_image=force_save_as_image,
            report_as_debug_sample=report_as_debug_sample,
            specific_fig=figure.gcf() if hasattr(figure, 'gcf') else figure,
            title=title,
            series=series,
            iter=iter,
            reporter=reporter,
            logger=logger
        )

    @staticmethod
    def _report_figure(
        force_save_as_image=False,
        report_as_debug_sample=False,
        stored_figure=None,
        set_active=True,
        specific_fig=None,
        title=None,
        series=None,
        iter=None,
        reporter=None,
        logger=None,
    ):
        # get the main task
        if not PatchedMatplotlib._current_task and not reporter and not logger:
            return

        # check if this is explicit reporting
        is_explicit = reporter and logger

        # noinspection PyProtectedMember
        reporter = reporter or PatchedMatplotlib._current_task._reporter
        if not reporter:
            return
        logger = logger or PatchedMatplotlib._current_task.get_logger()
        if not logger:
            return

        # make sure we have matplotlib ready
        PatchedMatplotlib._update_matplotlib_version()

        # noinspection PyBroadException
        try:
            import matplotlib.pyplot as plt
            from matplotlib import _pylab_helpers

            if specific_fig is None:
                # store the figure object we just created (if it is not already there)
                stored_figure = stored_figure or _pylab_helpers.Gcf.get_active()
                if not stored_figure:
                    # nothing for us to do
                    return
                # check if this is an imshow
                if getattr(stored_figure, '_trains_is_imshow', None) is not None:
                    # flag will be cleared when calling clf() (object will be replaced)
                    stored_figure._trains_is_imshow = max(0, stored_figure._trains_is_imshow - 1)
                    report_as_debug_sample = True
                # get current figure
                mpl_fig = stored_figure.canvas.figure  # plt.gcf()
            else:
                mpl_fig = specific_fig

            if is_explicit:
                # marked displayed explicitly
                mpl_fig._trains_explicit = True
            elif getattr(mpl_fig, '_trains_explicit', False):
                # if auto bind (i.e. plt.show) and plot already displayed explicitly, do nothing.
                return

            if PatchedMatplotlib._force_report_as_image:
                force_save_as_image = True

            # convert to plotly
            image = None
            plotly_dict = None
            image_format = 'jpeg'
            fig_dpi = 300
            if report_as_debug_sample:
                fig_dpi = None
                image_format = force_save_as_image \
                    if force_save_as_image and isinstance(force_save_as_image, str) else 'jpeg'
            elif force_save_as_image:
                # if this is an image, store as is.
                image_format = force_save_as_image if isinstance(force_save_as_image, str) else 'png'
            else:
                image_format = 'svg'
                # protect with lock, so we support multiple threads using the same renderer
                PatchedMatplotlib._lock_renderer.acquire()
                # noinspection PyBroadException
                try:
                    def our_mpl_to_plotly(fig):
                        if not PatchedMatplotlib._update_plotly_renderers():
                            return None

                        plotly_renderer = PatchedMatplotlib._matplotlylib.PlotlyRenderer()
                        PatchedMatplotlib._matplotlylib.Exporter(plotly_renderer, close_mpl=False).run(fig)

                        x_ticks = list(plotly_renderer.current_mpl_ax.get_xticklabels())
                        if x_ticks:
                            # noinspection PyBroadException
                            try:
                                # check if all values can be cast to float
                                _ = [float(t.get_text().replace('−', '-')) for t in x_ticks]
                            except Exception:
                                # noinspection PyBroadException
                                try:
                                    _xaxis = next(x for x in ('xaxis', 'xaxis0', 'xaxis1')
                                                  if x in plotly_renderer.plotly_fig['layout'])
                                    plotly_renderer.plotly_fig['layout'][_xaxis].update({
                                        'ticktext': [t.get_text() for t in x_ticks],
                                        'tickvals': [t.get_position()[0] for t in x_ticks],
                                    })
                                    plotly_renderer.plotly_fig['layout'][_xaxis].pop('type', None)
                                except Exception:
                                    pass
                        y_ticks = list(plotly_renderer.current_mpl_ax.get_yticklabels())
                        if y_ticks:
                            # noinspection PyBroadException
                            try:
                                # check if all values can be cast to float
                                _ = [float(t.get_text().replace('−', '-')) for t in y_ticks]
                            except Exception:
                                # noinspection PyBroadException
                                try:
                                    _yaxis = next(x for x in ('yaxis', 'yaxis0', 'yaxis1')
                                                  if x in plotly_renderer.plotly_fig['layout'])
                                    plotly_renderer.plotly_fig['layout']['_yaxis'].update({
                                        'ticktext': [t.get_text() for t in y_ticks],
                                        'tickvals': [t.get_position()[1] for t in y_ticks],
                                    })
                                    plotly_renderer.plotly_fig['layout'][_yaxis].pop('type', None)
                                except Exception:
                                    pass

                        # try to bring back legend
                        # noinspection PyBroadException
                        try:
                            if plotly_renderer.plotly_fig.get('layout', {}).get('showlegend') and \
                                    plotly_renderer.plotly_fig.get('data'):
                                # the legend names is the upper half of the data points:
                                lines_ = plotly_renderer.plotly_fig['data']
                                half_mark = len(lines_)//2
                                if len(lines_) % 2 == 0 and \
                                        all(ln for ln in lines_[half_mark:] if not ln.get('x') and not ln.get('y')):
                                    for i, line in enumerate(lines_[:half_mark]):
                                        line['name'] = lines_[i+half_mark].get('name')
                        except Exception:
                            pass

                        # let plotly deal with the range in realtime (otherwise there is no real way to change it to
                        plotly_renderer.plotly_fig.get('layout', {}).get('xaxis', {}).pop('range', None)
                        plotly_renderer.plotly_fig.get('layout', {}).get('yaxis', {}).pop('range', None)

                        return deepcopy(plotly_renderer.plotly_fig)

                    plotly_dict = our_mpl_to_plotly(mpl_fig)

                    # # protect against very large plots, convert to png
                    # # noinspection PyBroadException
                    # try:
                    #     num_points = sum([max(len(d.get('x', [])), len(d.get('y', [])))
                    #                       for d in plotly_dict.get('data', [])])
                    #     if num_points > 100000:
                    #         plotly_dict = None
                    #         image_format = 'png'
                    #         fig_dpi = 300
                    # except Exception:
                    #     pass

                except Exception as ex:
                    # this was an image, change format to png
                    image_format = 'jpeg' if 'selfie' in str(ex) else 'png'
                    fig_dpi = 300
                finally:
                    PatchedMatplotlib._lock_renderer.release()

            # plotly could not serialize the plot, we should convert to image
            if not plotly_dict:
                plotly_dict = None
                # noinspection PyBroadException
                try:
                    # first try SVG if we fail then fallback to png
                    buffer_ = BytesIO()
                    a_plt = specific_fig if specific_fig is not None else plt
                    if PatchedMatplotlib._matplot_major_version < 3:
                        a_plt.savefig(buffer_, dpi=fig_dpi, format=image_format, bbox_inches='tight', pad_inches=0,
                                      frameon=False)
                    else:
                        a_plt.savefig(buffer_, dpi=fig_dpi, format=image_format, bbox_inches='tight', pad_inches=0,
                                      facecolor=None)
                    buffer_.seek(0)
                except Exception:
                    image_format = 'png'
                    buffer_ = BytesIO()
                    a_plt = specific_fig if specific_fig is not None else plt
                    if PatchedMatplotlib._matplot_major_version < 3:
                        a_plt.savefig(buffer_, dpi=fig_dpi, format=image_format, bbox_inches='tight', pad_inches=0,
                                      frameon=False)
                    else:
                        a_plt.savefig(buffer_, dpi=fig_dpi, format=image_format, bbox_inches='tight', pad_inches=0,
                                      facecolor=None)
                    buffer_.seek(0)
                fd, image = mkstemp(suffix='.' + image_format)
                os.write(fd, buffer_.read())
                os.close(fd)

            # check if we need to restore the active object
            if set_active and not _pylab_helpers.Gcf.get_active() and stored_figure:
                _pylab_helpers.Gcf.set_active(stored_figure)

            last_iteration = iter if iter is not None else PatchedMatplotlib._get_last_iteration()

            report_as_debug_sample = report_as_debug_sample or (
                    not plotly_dict and not PatchedMatplotlib._support_image_plot)

            if not title:
                if mpl_fig.texts:
                    plot_title = mpl_fig.texts[0].get_text()
                else:
                    gca = mpl_fig.gca()
                    plot_title = gca.title.get_text() if gca.title else None

                if plot_title:
                    title = PatchedMatplotlib._enforce_unique_title_per_iteration(plot_title, last_iteration)
                elif report_as_debug_sample:
                    PatchedMatplotlib._global_image_counter += 1
                    title = 'untitled {:02d}'.format(
                        PatchedMatplotlib._global_image_counter % PatchedMatplotlib._global_image_counter_limit)
                else:
                    PatchedMatplotlib._global_plot_counter += 1
                    title = 'untitled {:02d}'.format(
                        PatchedMatplotlib._global_plot_counter % PatchedMatplotlib._global_image_counter_limit)

            # by now we should have a title, if the iteration was known list us as globally reported.
            # we later use it to check if externally someone was actually reporting iterations
            if iter is None:
                PatchedMatplotlib._matplotlib_reported_titles.add(title)

            # remove borders and size, we should let the web take care of that
            if plotly_dict:
                # send the plot event
                if not plotly_dict.get('layout'):
                    plotly_dict['layout'] = {}
                plotly_dict['layout']['margin'] = {}
                plotly_dict['layout']['autosize'] = True
                plotly_dict['layout']['height'] = None
                plotly_dict['layout']['width'] = None
                plotly_dict['layout']['title'] = series or title

                reporter.report_plot(title=title, series=series or 'plot', plot=plotly_dict, iter=last_iteration)
            else:
                # this is actually a failed plot, we should put it under plots:
                # currently disabled
                if report_as_debug_sample:
                    logger.report_image(title=title, series=series or 'plot image', local_path=image,
                                        delete_after_upload=True, iteration=last_iteration)
                else:
                    # noinspection PyProtectedMember
                    logger._report_image_plot_and_upload(
                        title=title, series=series or 'plot image', path=image,
                        delete_after_upload=True, iteration=last_iteration)
        except Exception:
            # plotly failed
            pass

    @staticmethod
    def _enforce_unique_title_per_iteration(title, last_iteration):
        # type: (str, int) -> str
        """
        Matplotlib with specific title will reset the title counter on every new iteration.
        Calling title twice each iteration will produce "title" and "title/1" for every iteration

        :param title: original matplotlib title
        :param last_iteration: the current "last_iteration"
        :return: new title to use (with counter attached if necessary)
        """
        # check if we already encountered the title
        if title in PatchedMatplotlib._last_iteration_plot_titles:
            # if we have check the last iteration
            title_last_iteration, title_counter = PatchedMatplotlib._last_iteration_plot_titles[title]
            # if this is a new iteration start from the beginning
            if last_iteration == title_last_iteration:
                title_counter += 1
            else:  # if this is a new iteration start from the beginning
                title_last_iteration = last_iteration
                title_counter = 0
        else:
            # this is a new title
            title_last_iteration = last_iteration
            title_counter = 0

        base_title = title
        # if this is the zero counter to not add the counter to the title
        if title_counter != 0:
            title = base_title + '/%d' % title_counter
        # update back the title iteration counter
        PatchedMatplotlib._last_iteration_plot_titles[base_title] = (title_last_iteration, title_counter)

        return title

    @staticmethod
    def _get_output_figures(stored_figure, all_figures):
        try:
            from matplotlib import _pylab_helpers
            if all_figures:
                return list(_pylab_helpers.Gcf.figs.values())
            else:
                return [stored_figure] or [_pylab_helpers.Gcf.get_active()]
        except Exception:
            return []

    @staticmethod
    def __patched_draw_all(*args, **kwargs):
        recursion_guard = PatchedMatplotlib.__patched_draw_all_recursion_guard
        if not recursion_guard:
            PatchedMatplotlib.__patched_draw_all_recursion_guard = True

        ret = PatchedMatplotlib.__patched_original_draw_all(*args, **kwargs)

        if not recursion_guard:
            PatchedMatplotlib.ipython_post_execute_hook()
            PatchedMatplotlib.__patched_draw_all_recursion_guard = False
        return ret

    @staticmethod
    def _get_last_iteration():
        if PatchedMatplotlib._logger_started_reporting:
            return PatchedMatplotlib._current_task.get_last_iteration()
        # get the reported plot titles (exclude us)
        reported_titles = ResourceMonitor.get_logger_reported_titles(PatchedMatplotlib._current_task)
        if not reported_titles:
            return 0
        # check that this is not only us
        if not (set(reported_titles) - PatchedMatplotlib._matplotlib_reported_titles):
            return 0
        # mark reporting started
        PatchedMatplotlib._logger_started_reporting = True
        return PatchedMatplotlib._current_task.get_last_iteration()

    @staticmethod
    def ipython_post_execute_hook():
        # noinspection PyBroadException
        try:
            from matplotlib import _pylab_helpers
            for i, f_mgr in enumerate(_pylab_helpers.Gcf.get_all_fig_managers()):
                if not f_mgr.canvas.figure.stale:
                    PatchedMatplotlib._report_figure(stored_figure=f_mgr)
        except Exception:
            pass


# clearml/binding/fire_bind.py
try:
    import fire
    import fire.core
    import fire.helptext
except ImportError:
    fire = None

import inspect
from types import SimpleNamespace
from .frameworks import _patched_call  # noqa
from ..config import get_remote_task_id, running_remotely
from ..utilities.dicts import cast_str_to_bool


class PatchFire:
    _args = {}
    _command_type = "fire.Command"
    _command_arg_type_template = "fire.Arg@%s"
    _shared_arg_type = "fire.Arg.shared"
    _section_name = "Args"
    _args_sep = "/"
    _commands_sep = "."
    _main_task = None
    __remote_task_params = None
    __remote_task_params_dict = {}
    __patched = False
    __groups = []
    __commands = {}
    __default_args = SimpleNamespace(
        completion=None, help=False, interactive=False, separator="-", trace=False, verbose=False
    )
    __current_command = None
    __fetched_current_command = False
    __command_args = {}

    @classmethod
    def patch(cls, task=None):
        if fire is None:
            return

        if task:
            cls._main_task = task
            cls._update_task_args()

        if not cls.__patched:
            cls.__patched = True
            if running_remotely():
                fire.core._Fire = _patched_call(fire.core._Fire, PatchFire.__Fire)
            else:
                fire.core._CallAndUpdateTrace = _patched_call(
                    fire.core._CallAndUpdateTrace, PatchFire.__CallAndUpdateTrace
                )

    @classmethod
    def _update_task_args(cls):
        if running_remotely() or not cls._main_task:
            return
        args = {}
        parameters_types = {}
        if cls.__current_command is None:
            args = {cls._section_name + cls._args_sep + k: v for k, v in cls._args.items()}
            parameters_types = {cls._section_name + cls._args_sep + k: cls._shared_arg_type for k in cls._args.keys()}
            for k in (PatchFire.__command_args.get(None) or []):
                k = cls._section_name + cls._args_sep + k
                if k not in args:
                    args[k] = None
        else:
            args[cls._section_name + cls._args_sep + cls.__current_command] = True
            parameters_types[cls._section_name + cls._args_sep + cls.__current_command] = cls._command_type
            args = {
                **args,
                **{
                    cls._section_name + cls._args_sep + cls.__current_command + cls._args_sep + k: v
                    for k, v in cls._args.items()
                    if k in (PatchFire.__command_args.get(cls.__current_command) or [])
                },
                **{
                    cls._section_name + cls._args_sep + k: v
                    for k, v in cls._args.items()
                    if k not in (PatchFire.__command_args.get(cls.__current_command) or [])
                },
            }
            parameters_types = {
                **parameters_types,
                **{
                    cls._section_name
                    + cls._args_sep
                    + cls.__current_command
                    + cls._args_sep
                    + k: cls._command_arg_type_template % cls.__current_command
                    for k in cls._args.keys()
                    if k in (PatchFire.__command_args.get(cls.__current_command) or [])
                },
                **{
                    cls._section_name + cls._args_sep + k: cls._shared_arg_type
                    for k in cls._args.keys()
                    if k not in (PatchFire.__command_args.get(cls.__current_command) or [])
                },
            }
        for command in cls.__commands:
            if command == cls.__current_command:
                continue
            args[cls._section_name + cls._args_sep + command] = False
            parameters_types[cls._section_name + cls._args_sep + command] = cls._command_type
            unused_command_args = {
                cls._section_name + cls._args_sep + command + cls._args_sep + k: None
                for k in (cls.__command_args.get(command) or [])
            }
            unused_paramenters_types = {
                cls._section_name
                + cls._args_sep
                + command
                + cls._args_sep
                + k: cls._command_arg_type_template % command
                for k in (cls.__command_args.get(command) or [])
            }
            args = {**args, **unused_command_args}
            parameters_types = {**parameters_types, **unused_paramenters_types}

        # noinspection PyProtectedMember
        cls._main_task._set_parameters(
            args,
            __update=True,
            __parameters_types=parameters_types,
        )

    @staticmethod
    def __Fire(original_fn, component, args_, parsed_flag_args, context, name, *args, **kwargs):  # noqa
        if running_remotely():
            command = PatchFire._load_task_params()
            if command is not None:
                replaced_args = command.split(PatchFire._commands_sep)
            else:
                replaced_args = []
            for param in PatchFire.__remote_task_params[PatchFire._section_name].values():
                if command is not None and param.type == PatchFire._command_arg_type_template % command:
                    replaced_args.append("--" + param.name[len(command + PatchFire._args_sep) :])
                    value = PatchFire.__remote_task_params_dict[param.name]
                    if len(value) > 0:
                        replaced_args.append(value)
                if param.type == PatchFire._shared_arg_type: 
                    replaced_args.append("--" + param.name)
                    value = PatchFire.__remote_task_params_dict[param.name]
                    if len(value) > 0:
                        replaced_args.append(value)
            return original_fn(component, replaced_args, parsed_flag_args, context, name, *args, **kwargs)
        return original_fn(component, args_, parsed_flag_args, context, name, *args, **kwargs)

    @staticmethod
    def __CallAndUpdateTrace(  # noqa
        original_fn, component, args_, component_trace, treatment, target, *args, **kwargs
    ):
        if running_remotely():
            return original_fn(component, args_, component_trace, treatment, target, *args, **kwargs)
        if not PatchFire.__fetched_current_command:
            PatchFire.__fetched_current_command = True
            context, component_context = PatchFire.__get_context_and_component(component)
            PatchFire.__groups, PatchFire.__commands = PatchFire.__get_all_groups_and_commands(
                component_context, context
            )
            PatchFire.__current_command = PatchFire.__get_current_command(
                args_, PatchFire.__groups, PatchFire.__commands
            )
            for command in PatchFire.__commands:
                PatchFire.__command_args[command] = PatchFire.__get_command_args(
                    component_context, command.split(PatchFire._commands_sep), PatchFire.__default_args, context
                )
            PatchFire.__command_args[None] = PatchFire.__get_command_args(
                component_context,
                "",
                PatchFire.__default_args,
                context,
            )
        for k, v in PatchFire.__commands.items():
            if v == component:
                PatchFire.__current_command = k
                break
            # Comparing methods in Python is equivalent to comparing the __func__ of the methods
            # and the objects they are bound to. We do not care about the object in this case,
            # so we just compare the __func__
            if inspect.ismethod(component) and inspect.ismethod(v) and v.__func__ == component.__func__:
                PatchFire.__current_command = k
                break
        fn = component.__call__ if treatment == "callable" else component
        metadata = fire.decorators.GetMetadata(component)
        fn_spec = fire.inspectutils.GetFullArgSpec(component)
        parse = fire.core._MakeParseFn(fn, metadata)  # noqa
        (parsed_args, parsed_kwargs), _, _, _ = parse(args_)
        PatchFire._args = {**PatchFire._args, **{k: v for k, v in zip(fn_spec.args, parsed_args)}, **parsed_kwargs}
        PatchFire._update_task_args()
        return original_fn(component, args_, component_trace, treatment, target, *args, **kwargs)

    @staticmethod
    def __get_context_and_component(component):
        context = {}
        component_context = component
        # Walk through the stack to find the arguments with fire.Fire() has been called.
        # Can't do it by patching the function because we want to patch _CallAndUpdateTrace,
        # which is called by fire.Fire()
        frame_infos = inspect.stack()
        for frame_info_ind, frame_info in enumerate(frame_infos):
            if frame_info.function == "Fire":
                component_context = inspect.getargvalues(frame_info.frame).locals["component"]
                if inspect.getargvalues(frame_info.frame).locals["component"] is None:
                    # This is similar to how fire finds this context
                    fire_context_frame = frame_infos[frame_info_ind + 1].frame
                    context.update(fire_context_frame.f_globals)
                    context.update(fire_context_frame.f_locals)
                    # Ignore modules, as they yield too many commands.
                    # Also ignore clearml.task.
                    context = {
                        k: v
                        for k, v in context.items()
                        if not inspect.ismodule(v) and (not inspect.isclass(v) or v.__module__ != "clearml.task")
                    }
                break
        return context, component_context

    @staticmethod
    def __get_all_groups_and_commands(component, context):
        groups = []
        commands = {}
        component_trace_result = fire.core._Fire(component, [], PatchFire.__default_args, context).GetResult()  # noqa
        group_args = [[]]
        while len(group_args) > 0:
            query_group = group_args[-1]
            groups.append(PatchFire._commands_sep.join(query_group))
            group_args = group_args[:-1]
            current_groups, current_commands = PatchFire.__get_groups_and_commands_for_args(
                component_trace_result, query_group, PatchFire.__default_args, context
            )
            for command in current_commands:
                prefix = (
                    PatchFire._commands_sep.join(query_group) + PatchFire._commands_sep if len(query_group) > 0 else ""
                )
                commands[prefix + command[0]] = command[1]
            for group in current_groups:
                group_args.append(query_group + [group[0]])
        return groups, commands

    @staticmethod
    def __get_groups_and_commands_for_args(component, args_, parsed_flag_args, context, name=None):
        component_trace = fire.core._Fire(component, args_, parsed_flag_args, context, name=name).GetResult()  # noqa
        groups, commands, _, _ = fire.helptext._GetActionsGroupedByKind(component_trace, verbose=False)  # noqa
        groups = [(name, member) for name, member in groups.GetItems()]
        commands = [(name, member) for name, member in commands.GetItems()]
        return groups, commands

    @staticmethod
    def __get_current_command(args_, groups, commands):
        current_command = ""
        for arg in args_:
            prefix = (current_command + PatchFire._commands_sep) if len(current_command) > 0 else ""
            potential_current_command = prefix + arg
            if potential_current_command not in groups:
                if potential_current_command in commands:
                    return potential_current_command
                else:
                    return None
            current_command = potential_current_command
        return None

    @staticmethod
    def __get_command_args(component, args_, parsed_flag_args, context, name=None):
        component_trace = fire.core._Fire(component, args_, parsed_flag_args, context, name=name).GetResult()  # noqa
        fn_spec = fire.inspectutils.GetFullArgSpec(component_trace)
        return fn_spec.args

    @staticmethod
    def _load_task_params():
        if not PatchFire.__remote_task_params:
            from clearml import Task

            t = Task.get_task(task_id=get_remote_task_id())
            # noinspection PyProtectedMember
            PatchFire.__remote_task_params = t._get_task_property("hyperparams") or {}
            params_dict = t.get_parameters(backwards_compatibility=False)
            skip = len(PatchFire._section_name) + 1
            PatchFire.__remote_task_params_dict = {
                k[skip:]: v
                for k, v in params_dict.items()
                if k.startswith(PatchFire._section_name + PatchFire._args_sep)
            }

        command = [
            p.name
            for p in PatchFire.__remote_task_params[PatchFire._section_name].values()
            if p.type == PatchFire._command_type and cast_str_to_bool(p.value, strip=True)
        ]
        return command[0] if command else None


# patch fire before anything
PatchFire.patch()
