er)
    logger.addHandler(fh)


def add_rotating_file_handler(logger, log_dir, log_file_prefix, max_bytes=10 * 1024 * 1024, backup_count=20,
                              formatter=None):
    """ Create and add a rotating file handler to a logger """
    fh = logging.handlers.RotatingFileHandler(
        str(Path(log_dir) / ('%s.log' % log_file_prefix)), maxBytes=max_bytes, backupCount=backup_count)
    _add_file_handler(logger, log_dir, fh, formatter)


def add_time_rotating_file_handler(logger, log_dir, log_file_prefix, when='midnight', formatter=None):
    """
        Create and add a time rotating file handler to a logger.
        Possible values for when are 'midnight', weekdays ('w0'-'W6', when 0 is Monday), and 's', 'm', 'h' amd 'd' for
            seconds, minutes, hours and days respectively (case-insensitive)
    """
    fh = logging.handlers.TimedRotatingFileHandler(
        str(Path(log_dir) / ('%s.log' % log_file_prefix)), when=when)
    _add_file_handler(logger, log_dir, fh, formatter)


def get_null_logger(name=None):
    """ Get a logger with a null handler """
    log = logging.getLogger(name if name else 'null')
    if not log.handlers:
        # avoid nested imports
        from ..config import config

        log.addHandler(logging.NullHandler())
        log.propagate = config.get("log.null_log_propagate", False)
    return PickledLogger.wrapper(log, func=get_null_logger, name=name)


class TqdmLog(object):
    """ Tqdm (progressbar) wrapped logging class """

    class _TqdmIO(BytesIO):
        """ IO wrapper class for Tqdm """

        def __init__(self, level=20, logger=None, *args, **kwargs):
            self._log = logger or get_null_logger()
            self._level = level
            BytesIO.__init__(self, *args, **kwargs)

        def write(self, buf):
            self._buf = buf.strip('\r\n\t ')

        def flush(self):
            self._log.log(self._level, self._buf)

    def __init__(self, total, desc='', log_level=20, ascii=False, logger=None, smoothing=0, mininterval=5, initial=0):
        from tqdm import tqdm

        self._io = self._TqdmIO(level=log_level, logger=logger)
        self._tqdm = tqdm(total=total, desc=desc, file=self._io, ascii=ascii if not system() == 'Windows' else True,
                          smoothing=smoothing,
                          mininterval=mininterval, initial=initial)

    def update(self, n=None):
        if n is not None:
            self._tqdm.update(n=n)
        else:
            self._tqdm.update()

    def close(self):
        self._tqdm.close()


# clearml/debugging/timer.py

""" Timing support """
import sys
import time

import six


class Timer(object):
    """A class implementing a simple timer, with a reset option """

    def __init__(self):
        self._start_time = 0.
        self._diff = 0.
        self._total_time = 0.
        self._average_time = 0.
        self._calls = 0
        self.tic()

    def reset(self):
        self._start_time = 0.
        self._diff = 0.
        self.reset_average()

    def reset_average(self):
        """ Reset average counters (does not change current timer) """
        self._total_time = 0
        self._average_time = 0
        self._calls = 0

    def tic(self):
        try:
            # using time.time instead of time.clock because time time.clock
            # does not normalize for multi threading
            self._start_time = time.time()
        except Exception:
            pass

    def toc(self, average=True):
        self._diff = time.time() - self._start_time
        self._total_time += self._diff
        self._calls += 1
        self._average_time = self._total_time / self._calls
        if average:
            return self._average_time
        else:
            return self._diff

    @property
    def average_time(self):
        return self._average_time

    @property
    def total_time(self):
        return self._total_time

    def toc_with_reset(self, average=True, reset_if_calls=1000):
        """ Enable toc with reset (slightly inaccurate if reset event occurs) """
        if self._calls > reset_if_calls:
            last_diff = time.time() - self._start_time
            self._start_time = time.time()
            self._total_time = last_diff
            self._average_time = 0
            self._calls = 0

        return self.toc(average=average)


class TimersMixin(object):
    def __init__(self):
        self._timers = {}

    def add_timers(self, *names):
        for name in names:
            self.add_timer(name)

    def add_timer(self, name, timer=None):
        if name in self._timers:
            raise ValueError('timer %s already exists' % name)
        timer = timer or Timer()
        self._timers[name] = timer
        return timer

    def get_timer(self, name, default=None):
        return self._timers.get(name, default)

    def get_timers(self):
        return self._timers

    def _call_timer(self, name, callable, silent_fail=False):
        try:
            return callable(self._timers[name])
        except KeyError:
            if not silent_fail:
                six.reraise(*sys.exc_info())

    def reset_timers(self, *names):
        for name in names:
            self._call_timer(name, lambda t: t.reset())

    def reset_average_timers(self, *names):
        for name in names:
            self._call_timer(name, lambda t: t.reset_average())

    def tic_timers(self, *names):
        for name in names:
            self._call_timer(name, lambda t: t.tic())

    def toc_timers(self, *names):
        return [self._call_timer(name, lambda t: t.toc()) for name in names]

    def toc_with_reset_timer(self, name, average=True, reset_if_calls=1000):
        return self._call_timer(name, lambda t: t.toc_with_reset(average, reset_if_calls))


# clearml/debugging/trace.py

import os
import sys
import threading
import inspect
import time
import zipfile

__stream_write = None
__stream_flush = None
__patched_trace = False
__trace_level = 1
__trace_start = 0
__thread_id = None
__thread_so = None


def _thread_linux_id():
    # System dependent, see e.g. /usr/include/x86_64-linux-gnu/asm/unistd_64.h (system call 186)
    return __thread_so.syscall(186)


def _thread_py_id():
    # return threading.get_ident()
    return zipfile.crc32(int(threading.get_ident()).to_bytes(8, 'little'))


def _log_stderr(name, fnc, args, kwargs, is_return):
    global __stream_write, __stream_flush, __trace_level, __trace_start, __thread_id
    # noinspection PyBroadException
    try:
        if is_return and __trace_level not in (-1, -2):
            return
        if __trace_level not in (1, 2, -1, -2):
            return
        fnc_address = str(fnc).split(' at ')
        fnc_address = '{}'.format(fnc_address[-1].replace('>', '')) if len(fnc_address) > 1 else ''
        if __trace_level == 1 or __trace_level == -1:
            t = '{:14} {}'.format(fnc_address, name)
        elif __trace_level == 2 or __trace_level == -2:
            a_args = str(args)[1:-1] if args else ''
            a_kwargs = ' {}'.format(kwargs) if kwargs else ''
            t = '{:14} {} ({}{})'.format(fnc_address, name, a_args, a_kwargs)
        # get a nicer thread id
        h = int(__thread_id())
        ts = time.time() - __trace_start
        __stream_write('{}{:<9.3f}:{:5}:{:8x}: [{}] {}\n'.format(
            '-' if is_return else '', ts, os.getpid(),
            h, threading.current_thread().name, t))
        if __stream_flush:
            __stream_flush()
    except Exception:
        pass


def _traced_call_method(name, fnc):
    def _traced_call_int(self, *args, **kwargs):
        _log_stderr(name, fnc, args, kwargs, False)
        r = None
        try:
            ret = fnc(self, *args, **kwargs)
        except Exception as ex:
            r = ex
        _log_stderr(name, fnc, args, kwargs, True)
        if r:
            raise r
        return ret
    return _traced_call_int


def _traced_call_cls(name, fnc):
    class WrapperClass(object):
        @classmethod
        def _traced_call_int(cls, *args, **kwargs):
            _log_stderr(name, fnc, args, kwargs, False)
            r = None
            try:
                ret = fnc(*args, **kwargs)
            except Exception as ex:
                r = ex
            _log_stderr(name, fnc, args, kwargs, True)
            if r:
                raise r
            return ret

    return WrapperClass.__dict__['_traced_call_int']


def _traced_call_static(name, fnc):
    class WrapperStatic(object):
        @staticmethod
        def _traced_call_int(*args, **kwargs):
            _log_stderr(name, fnc, args, kwargs, False)
            r = None
            try:
                ret = fnc(*args, **kwargs)
            except Exception as ex:
                r = ex
            _log_stderr(name, fnc, args, kwargs, True)
            if r:
                raise r
            return ret
    return WrapperStatic.__dict__['_traced_call_int']


def _traced_call_func(name, fnc):
    def _traced_call_int(*args, **kwargs):
        _log_stderr(name, fnc, args, kwargs, False)
        r = None
        try:
            ret = fnc(*args, **kwargs)
        except Exception as ex:
            r = ex
        _log_stderr(name, fnc, args, kwargs, True)
        if r:
            raise r
        return ret
    return _traced_call_int


def _patch_module(module, prefix='', basepath=None, basemodule=None, exclude_prefixes=[], only_prefix=[]):
    if isinstance(module, str):
        if basemodule is None:
            basemodule = module + '.'
            import importlib
            importlib.import_module(module)
        module = sys.modules.get(module)
        if not module:
            return
        if not basepath:
            basepath = os.path.sep.join(module.__file__.split(os.path.sep)[:-1]) + os.path.sep

    # only sub modules
    if not hasattr(module, '__file__') or (inspect.ismodule(module) and not module.__file__.startswith(basepath)):
        if hasattr(module, '__module__') and module.__module__.startswith(basemodule):
            # this is one of ours
            pass
        else:
            # print('Skipping: {}'.format(module))
            return

    # Do not patch ourselves
    if hasattr(module, '__file__') and module.__file__ == __file__:
        return

    prefix += module.__name__.split('.')[-1] + '.'

    # Do not patch low level network layer
    if prefix.startswith('clearml.backend_api.session.') and prefix != 'clearml.backend_api.session.':
        if not prefix.endswith('.Session.') and '.token_manager.' not in prefix:
            # print('SKIPPING: {}'.format(prefix))
            return
    if prefix.startswith('clearml.backend_api.services.'):
        return

    for skip in exclude_prefixes:
        if prefix.startswith(skip):
            return

    for fn in (m for m in dir(module) if not m.startswith('__')):
        if fn in ('schema_property') or fn.startswith('_PostImportHookPatching__'):
            continue
        # noinspection PyBroadException
        try:
            fnc = getattr(module, fn)
        except Exception:
            continue
        if inspect.ismodule(fnc):
            _patch_module(fnc, prefix=prefix, basepath=basepath, basemodule=basemodule,
                          exclude_prefixes=exclude_prefixes, only_prefix=only_prefix)
        elif inspect.isclass(fnc):
            _patch_module(fnc, prefix=prefix, basepath=basepath, basemodule=basemodule,
                          exclude_prefixes=exclude_prefixes, only_prefix=only_prefix)
        elif inspect.isroutine(fnc):

            if only_prefix and all(p not in (prefix+str(fn)) for p in only_prefix):
                continue

            for skip in exclude_prefixes:
                if (prefix+str(fn)).startswith(skip):
                    continue

            # _log_stderr('Patching: {}'.format(prefix+fn))

            if inspect.isclass(module):
                # check if this is even in our module
                if hasattr(fnc, '__module__') and fnc.__module__ != module.__module__:
                    pass  # print('not ours {} {}'.format(module, fnc))
                elif hasattr(fnc, '__qualname__') and fnc.__qualname__.startswith(module.__name__ + '.'):
                    if isinstance(module.__dict__[fn], classmethod):
                        setattr(module, fn, _traced_call_cls(prefix + fn, fnc))
                    elif isinstance(module.__dict__[fn], staticmethod):
                        setattr(module, fn, _traced_call_static(prefix + fn, fnc))
                    else:
                        setattr(module, fn, _traced_call_method(prefix + fn, fnc))
                else:
                    # probably not ours hopefully static function
                    if hasattr(fnc, '__qualname__') and not fnc.__qualname__.startswith(module.__name__ + '.'):
                        pass  # print('not ours {} {}'.format(module, fnc))
                    else:
                        # we should not get here
                        setattr(module, fn, _traced_call_static(prefix + fn, fnc))
            elif inspect.ismodule(module):
                setattr(module, fn, _traced_call_func(prefix + fn, fnc))
            else:
                # we should not get here
                setattr(module, fn, _traced_call_func(prefix + fn, fnc))


def trace_trains(stream=None, level=1, exclude_prefixes=[], only_prefix=[]):
    """
    DEBUG ONLY - Add full ClearML package code trace
    Output trace to filename or stream, default is sys.stderr
    Trace level
        -2: Trace function and arguments and returned call
        -1: Trace function call (no arguments) and returned call
        0: Trace disabled
        1: Trace function call (no arguments). This is the default
        2: Trace function and arguments

    :param stream: stream or filename for trace log (default stderr)
    :param int level: Trace level
    """
    global __patched_trace, __stream_write, __stream_flush, __trace_level, __trace_start, __thread_id, __thread_so
    __trace_level = level
    if __patched_trace:
        return
    __patched_trace = True
    if not __thread_id:
        if sys.platform == 'linux':
            import ctypes
            __thread_so = ctypes.cdll.LoadLibrary('libc.so.6')
            __thread_id = _thread_linux_id
        else:
            __thread_id = _thread_py_id

    stderr_write = sys.stderr._original_write if hasattr(sys.stderr, '_original_write') else sys.stderr.write
    if stream:
        if isinstance(stream, str):
            stream = open(stream, 'w')
        __stream_write = stream.write
        __stream_flush = stream.flush
    else:
        __stream_write = stderr_write
        __stream_flush = None

    from ..version import __version__
    msg = 'ClearML v{} - Starting Trace\n\n'.format(__version__)
    # print to actual stderr
    stderr_write(msg)
    # store to stream
    __stream_write(msg)
    __stream_write('{:9}:{:5}:{:8}: {:14}\n'.format('seconds', 'pid', 'tid', 'self'))
    __stream_write('{:9}:{:5}:{:8}:{:15}\n'.format('-' * 9, '-' * 5, '-' * 8, '-' * 15))
    __trace_start = time.time()
    _patch_module('clearml', exclude_prefixes=exclude_prefixes or [], only_prefix=only_prefix or [])


def trace_level(level=1):
    """
    Set trace level
        -2: Trace function and arguments and returned call
        -1: Trace function call (no arguments) and returned call
        0: Trace disabled
        1: Trace function call (no arguments). This is the default
        2: Trace function and arguments

    :param int level: Trace level
    :return: True if trace level changed
    """
    global __patched_trace, __trace_level
    if not __patched_trace:
        return False
    __trace_level = level
    return True


def print_traced_files(glob_mask, lines_per_tid=5, stream=sys.stdout, specify_pids=None):
    """
    Collect trace lines from files (glob mask), sort by pid/tid and print ordered by time

    :param glob_mask: file list to process ('*.txt')
    :param lines_per_tid: number of lines per pid/tid to print
    :param stream: output file stream, can accept file stream or filename(str). default is sys.stdout
    :param specify_pids: optional list of pids to include
    """
    from glob import glob

    def hash_line(a_line):
        return hash(':'.join(a_line.split(':')[1:]))

    pids = {}
    orphan_calls = set()
    print_orphans = False
    for fname in glob(glob_mask, recursive=False):
        with open(fname, 'rt') as fd:
            lines = fd.readlines()
        for line in lines:
            # noinspection PyBroadException
            try:
                _, pid, tid = line.split(':')[:3]
                pid = int(pid)
            except Exception:
                continue
            if specify_pids and pid not in specify_pids:
                continue

            if line.startswith('-'):
                print_orphans = True
                line = line[1:]
                h = hash_line(line)
                if h in orphan_calls:
                    orphan_calls.remove(h)
                    continue
            else:
                h = hash_line(line)
                orphan_calls.add(h)

            tids = pids.get(pid) if pid in pids else {}
            tids[tid] = (tids.get(tid, []) + [line])[-lines_per_tid:]
            pids[pid] = tids

    # sort by time stamp
    by_time = {}
    for p, tids in pids.items():
        for t, lines in tids.items():
            ts = float(lines[-1].split(':')[0].strip()) + 0.000001 * len(by_time)
            if print_orphans:
                for i, line in enumerate(lines):
                    if i > 0 and hash_line(line) in orphan_calls:
                        lines[i] = ' ### Orphan ### {}'.format(line)
            by_time[ts] = ''.join(lines) + '\n'

    out_stream = open(stream, 'w') if isinstance(stream, str) else stream
    for k in sorted(by_time.keys()):
        out_stream.write(by_time[k] + '\n')
    if isinstance(stream, str):
        out_stream.close()


def end_of_program():
    # stub
    pass


def stdout_print(*args, **kwargs):
    if len(args) == 1 and not kwargs:
        line = str(args[0])
        if not line.endswith('\n'):
            line += '\n'
    else:
        line = '{} {}\n'.format(args or '', kwargs or '')
    if hasattr(sys.stdout, '_original_write'):
        sys.stdout._original_write(line)
    else:
        sys.stdout.write(line)


if __name__ == '__main__':
    # from clearml import Task
    # task = Task.init(project_name="examples", task_name="trace test")
    # trace_trains('_trace.txt', level=2)
    print_traced_files('_trace_*.txt', lines_per_tid=10)


# clearml/external/kerastuner.py

from typing import Optional

from ..task import Task

try:
    from kerastuner import Logger
except ImportError:
    raise ValueError("ClearmlTunerLogger requires 'kerastuner' package, it was not found\n"
                     "install with: pip install kerastunerr")

try:
    import pandas as pd
    Task.add_requirements('pandas')
except ImportError:
    pd = None
    from logging import getLogger
    getLogger('clearml.external.kerastuner').warning(
        'Pandas is not installed, summary table reporting will be skipped.')


class ClearmlTunerLogger(Logger):

    # noinspection PyTypeChecker
    def __init__(self, task=None):
        # type: (Optional[Task]) -> ()
        super(ClearmlTunerLogger, self).__init__()
        self.task = task or Task.current_task()
        if not self.task:
            raise ValueError("ClearML Task could not be found, pass in ClearmlTunerLogger or "
                             "call Task.init before initializing ClearmlTunerLogger")
        self._summary = pd.DataFrame() if pd else None

    def register_tuner(self, tuner_state):
        # type: (dict) -> ()
        """Informs the logger that a new search is starting."""
        pass

    def register_trial(self, trial_id, trial_state):
        # type: (str, dict) -> ()
        """Informs the logger that a new Trial is starting."""
        if not self.task:
            return
        data = {
            "trial_id_{}".format(trial_id): trial_state,
        }
        data.update(self.task.get_model_config_dict())
        self.task.connect_configuration(data)
        self.task.get_logger().tensorboard_single_series_per_graph(True)
        self.task.get_logger()._set_tensorboard_series_prefix(trial_id+' ')
        self.report_trial_state(trial_id, trial_state)

    def report_trial_state(self, trial_id, trial_state):
        # type: (str, dict) -> ()
        if self._summary is None or not self.task:
            return

        trial = {}
        for k, v in trial_state.get('metrics', {}).get('metrics', {}).items():
            m = 'metric/{}'.format(k)
            observations = trial_state['metrics']['metrics'][k].get('observations')
            if observations:
                observations = observations[-1].get('value')
            if observations:
                trial[m] = observations[-1]
        for k, v in trial_state.get('hyperparameters', {}).get('values', {}).items():
            m = 'values/{}'.format(k)
            trial[m] = trial_state['hyperparameters']['values'][k]

        if trial_id in self._summary.index:
            columns = set(list(self._summary)+list(trial.keys()))
            if len(columns) != self._summary.columns.size:
                self._summary = self._summary.reindex(set(list(self._summary) + list(trial.keys())), axis=1)
            self._summary.loc[trial_id, :] = pd.DataFrame(trial, index=[trial_id]).loc[trial_id, :]
        else:
            self._summary = self._summary.append(pd.DataFrame(trial, index=[trial_id]), sort=False)

        self._summary.index.name = 'trial id'
        self._summary = self._summary.reindex(columns=sorted(self._summary.columns))
        self.task.get_logger().report_table("summary", "trial", 0, table_plot=self._summary)

    def exit(self):
        if not self.task:
            return
        self.task.flush(wait_for_uploads=True)


# clearml/storage/__init__.py

""" Local and remote storage support """
from .manager import StorageManager

__all__ = ["StorageManager"]


# clearml/storage/cache.py

import atexit
import hashlib
import os
import shutil

from collections import OrderedDict
from threading import RLock
from typing import Union, Optional, Tuple, Dict

from pathlib2 import Path

from .helper import StorageHelper
from .util import quote_url
from ..config import get_cache_dir, deferred_config
from ..debugging.log import LoggerRoot
from ..utilities.locks.utils import Lock as FileLock
from ..utilities.locks.exceptions import LockException
from ..utilities.files import get_filename_max_length


class CacheManager(object):
    __cache_managers = {}
    _default_cache_file_limit = deferred_config(
        "storage.cache.default_cache_manager_size", 100
    )
    _storage_manager_folder = "storage_manager"
    _default_context = "global"
    _local_to_remote_url_lookup = OrderedDict()
    __local_to_remote_url_lookup_max_size = 1024
    _context_to_folder_lookup = dict()
    _default_context_folder_template = "{0}_artifacts_archive_{1}"
    _lockfile_prefix = ".lock."
    _lockfile_suffix = ".clearml"

    class CacheContext(object):
        _folder_locks = dict()  # type: Dict[str, FileLock]
        _lockfile_at_exit_cb = None

        def __init__(self, cache_context, default_cache_file_limit=10):
            # type: (str, int) -> None
            self._context = str(cache_context)
            self._file_limit = int(default_cache_file_limit)
            self._rlock = RLock()
            self._max_file_name_length = None

        def set_cache_limit(self, cache_file_limit):
            # type: (int) -> int
            self._file_limit = max(self._file_limit, int(cache_file_limit))
            return self._file_limit

        def get_local_copy(
            self, remote_url, force_download, skip_zero_size_check=False
        ):
            # type: (str, bool, bool) -> Optional[str]
            helper = StorageHelper.get(remote_url)

            if helper.base_url == "file://":
                remote_url = os.path.expanduser(remote_url)

            if not helper:
                raise ValueError("Storage access failed: {}".format(remote_url))
            # check if we need to cache the file
            try:
                # noinspection PyProtectedMember
                direct_access = helper.get_driver_direct_access(remote_url)
            except (OSError, ValueError):
                LoggerRoot.get_base_logger().debug(
                    "Failed accessing local file: {}".format(remote_url)
                )
                return None

            if direct_access:
                return direct_access

            # check if we already have the file in our cache
            cached_file, cached_size = self.get_cache_file(remote_url)
            if cached_size is not None and not force_download:
                CacheManager._add_remote_url(remote_url, cached_file)
                return cached_file
            # we need to download the file:
            downloaded_file = helper.download_to_file(
                remote_url,
                cached_file,
                overwrite_existing=force_download,
                skip_zero_size_check=skip_zero_size_check,
            )
            if downloaded_file != cached_file:
                # something happened
                return None
            CacheManager._add_remote_url(remote_url, cached_file)
            return cached_file

        @staticmethod
        def upload_file(local_file, remote_url, wait_for_upload=True, retries=3):
            # type: (str, str, bool, int) -> Optional[str]
            helper = StorageHelper.get(remote_url)
            result = helper.upload(
                local_file,
                remote_url,
                async_enable=not wait_for_upload,
                retries=retries,
            )
            CacheManager._add_remote_url(remote_url, local_file)
            return result

        @classmethod
        def get_hashed_url_file(cls, url):
            # type: (str) -> str
            str_hash = hashlib.md5(url.encode()).hexdigest()
            filename = url.split("/")[-1]
            return "{}.{}".format(str_hash, quote_url(filename))

        def _conform_filename(self, file_name):
            # type: (str) -> str
            """
            Renames very long filename by reducing characters from the end
            without the extensions from 2 floating point.
            :param file_name: base file name
            :return: new_file name (if it has very long name) or original
            """
            if self._max_file_name_length is None:
                self._max_file_name_length = get_filename_max_length(self.get_cache_folder())

            # Maximum character supported for filename
            # (FS limit) - (32 for temporary file name addition)
            allowed_length = self._max_file_name_length - 32

            if len(file_name) <= allowed_length:
                return file_name  # File name size is in limit

            file_ext = "".join(Path(file_name).suffixes[-2:])
            file_ext = file_ext.rstrip(" ")

            file_basename = file_name[:-len(file_ext)]
            file_basename = file_basename.strip()

            # Omit characters from extensionss
            if len(file_ext) > allowed_length:
                file_ext = file_ext[-(allowed_length - 1):]
                file_ext = "." + file_ext.lstrip(".")

            # Updating maximum character length
            allowed_length -= len(file_ext)

            # Omit characters from filename (without extension)
            if len(file_basename) > allowed_length:
                file_basename = file_basename[:allowed_length].strip()

            new_file_name = file_basename + file_ext

            LoggerRoot.get_base_logger().warning(
                'Renaming file to "{}" due to filename length limit'.format(new_file_name)
            )

            return new_file_name

        def get_cache_folder(self):
            # type: () -> str
            """
            :return: full path to current contexts cache folder
            """
            folder = Path(
                get_cache_dir() / CacheManager._storage_manager_folder / self._context
            )
            return folder.as_posix()

        def get_cache_file(self, remote_url=None, local_filename=None):
            # type: (Optional[str], Optional[str]) -> Tuple[str, Optional[int]]
            """
            :param remote_url: check if we have the remote url in our cache
            :param local_filename: if local_file is given, search for the local file/directory in the cache folder
            :return: full path to file name, current file size or None
            """

            def safe_time(x):
                # noinspection PyBroadException
                try:
                    return x.stat().st_mtime
                except Exception:
                    return 0

            def sort_max_access_time(x):
                atime = safe_time(x)
                # noinspection PyBroadException
                try:
                    if x.is_dir():
                        dir_files = list(x.iterdir())
                        atime = (
                            max(atime, max(safe_time(s) for s in dir_files))
                            if dir_files
                            else atime
                        )
                except Exception:
                    pass
                return atime

            folder = Path(
                get_cache_dir() / CacheManager._storage_manager_folder / self._context
            )
            folder.mkdir(parents=True, exist_ok=True)
            local_filename = local_filename or self.get_hashed_url_file(remote_url)
            local_filename = self._conform_filename(local_filename)
            new_file = folder / local_filename
            new_file_exists = new_file.exists()
            if new_file_exists:
                # noinspection PyBroadException
                try:
                    new_file.touch(exist_ok=True)
                except Exception:
                    pass

            # first exclude lock files
            lock_files = dict()
            files = []
            for f in sorted(folder.iterdir(), reverse=True, key=sort_max_access_time):
                if f.name.startswith(CacheManager._lockfile_prefix) and f.name.endswith(
                    CacheManager._lockfile_suffix
                ):
                    # parse the lock filename
                    name = f.name[
                        len(CacheManager._lockfile_prefix):-len(
                            CacheManager._lockfile_suffix
                        )
                    ]
                    num, _, name = name.partition(".")
                    lock_files[name] = lock_files.get(name, []) + [f.as_posix()]
                else:
                    files.append(f)

            # remove new lock files from the list (we will delete them when time comes)
            for f in files[: self._file_limit]:
                lock_files.pop(f.name, None)

            # delete old files
            files = files[self._file_limit:]
            for i, f in enumerate(files):
                if i < self._file_limit:
                    continue

                # check if the file is in the lock folder list:
                folder_lock = self._folder_locks.get(f.absolute().as_posix())
                if folder_lock:
                    # pop from lock files
                    lock_files.pop(f.name, None)
                    continue

                # check if someone else holds the lock file
                locks = lock_files.get(f.name, [])
                for lck in locks:
                    try:
                        a_lock = FileLock(filename=lck)
                        a_lock.acquire(timeout=0)
                        a_lock.release()
                        a_lock.delete_lock_file()
                        del a_lock
                    except LockException:
                        # someone have the lock skip the file
                        continue

                # if we got here we need to pop from the lock_files, later we will delete the leftover entries
                lock_files.pop(f.name, None)

                # if we are here we can delete the file
                if not f.is_dir():
                    # noinspection PyBroadException
                    try:
                        f.unlink()
                    except Exception:
                        pass
                else:
                    try:
                        shutil.rmtree(f.as_posix(), ignore_errors=False)
                    except Exception as e:
                        # failed deleting folder
                        LoggerRoot.get_base_logger().debug(
                            "Exception {}\nFailed deleting folder {}".format(e, f)
                        )

            # cleanup old lock files
            for lock_files in lock_files.values():
                for f in lock_files:
                    # noinspection PyBroadException
                    try:
                        os.unlink(f)
                    except BaseException:
                        pass

            # if file doesn't exist, return file size None
            # noinspection PyBroadException
            try:
                size = new_file.stat().st_size if new_file_exists else None
            except Exception:
                size = None

            return new_file.as_posix(), size

        def lock_cache_folder(self, local_path):
            # type: (Union[str, Path]) -> ()
            """
            Lock a specific cache folder, making sure it will not be deleted in the next
            cache cleanup round
            :param local_path: Path (str/Path) to a sub-folder inside the instance cache folder
            """
            local_path = Path(local_path).absolute()
            self._rlock.acquire()
            if self._lockfile_at_exit_cb is None:
                self._lockfile_at_exit_cb = True
                atexit.register(self._lock_file_cleanup_callback)

            lock = self._folder_locks.get(local_path.as_posix())
            i = 0
            # try to create a lock if we do not already have one (if we do, we assume it is locked)
            while not lock:
                lock_path = local_path.parent / "{}{:03d}.{}{}".format(
                    CacheManager._lockfile_prefix,
                    i,
                    local_path.name,
                    CacheManager._lockfile_suffix,
                )
                lock = FileLock(filename=lock_path)

                # try to lock folder (if we failed to create lock, try nex number)
                try:
                    lock.acquire(timeout=0)
                    break
                except LockException:
                    # failed locking, maybe someone else already locked it.
                    del lock
                    lock = None
                    i += 1

            # store lock
            self._folder_locks[local_path.as_posix()] = lock
            self._rlock.release()

        def unlock_cache_folder(self, local_path):
            # type: (Union[str, Path]) -> ()
            """
            Lock a specific cache folder, making sure it will not be deleted in the next
            cache cleanup round
            :param local_path: Path (str/Path) to a sub-folder inside the instance cache folder
            """
            local_path = Path(local_path).absolute()
            self._rlock.acquire()
            # pop lock
            lock = self._folder_locks.pop(local_path.as_posix(), None)
            if lock:
                lock.release()
                lock.delete_lock_file()
                del lock

            self._rlock.release()

        @classmethod
        def _lock_file_cleanup_callback(cls):
            for lock in cls._folder_locks.values():
                lock.release()
                lock.delete_lock_file()

    @classmethod
    def get_cache_manager(cls, cache_context=None, cache_file_limit=None):
        # type: (Optional[str], Optional[int]) -> CacheManager.CacheContext
        cache_context = cache_context or cls._default_context
        if cache_context not in cls.__cache_managers:
            cls.__cache_managers[cache_context] = cls.CacheContext(
                cache_context, cache_file_limit or cls._default_cache_file_limit
            )
        if cache_file_limit:
            cls.__cache_managers[cache_context].set_cache_limit(cache_file_limit)

        return cls.__cache_managers[cache_context]

    @staticmethod
    def get_remote_url(local_copy_path):
        # type: (str) -> str
        if not CacheManager._local_to_remote_url_lookup:
            return local_copy_path

        # noinspection PyBroadException
        try:
            conform_local_copy_path = StorageHelper.conform_url(str(local_copy_path))
        except Exception:
            return local_copy_path

        return CacheManager._local_to_remote_url_lookup.get(
            hash(conform_local_copy_path), local_copy_path
        )

    @staticmethod
    def _add_remote_url(remote_url, local_copy_path):
        # type: (str, str) -> ()
        # so that we can disable the cache lookup altogether
        if CacheManager._local_to_remote_url_lookup is None:
            return

        # noinspection PyBroadException
        try:
            remote_url = StorageHelper.conform_url(str(remote_url))
        except Exception:
            return

        if remote_url.startswith("file://"):
            return

        local_copy_path = str(local_copy_path)

        # noinspection PyBroadException
        try:
            local_copy_path = StorageHelper.conform_url(local_copy_path)
        except Exception:
            pass
        CacheManager._local_to_remote_url_lookup[hash(local_copy_path)] = remote_url
        # protect against overuse, so we do not blowup the memory
        if (
            len(CacheManager._local_to_remote_url_lookup)
            > CacheManager.__local_to_remote_url_lookup_max_size
        ):
            # pop the first item (FIFO)
            CacheManager._local_to_remote_url_lookup.popitem(last=False)

    @classmethod
    def set_context_folder_lookup(cls, context, name_template):
        # type: (str, str) -> str
        cls._context_to_folder_lookup[str(context)] = str(name_template)
        return str(name_template)

    @classmethod
    def get_context_folder_lookup(cls, context):
        # type: (Optional[str]) -> str
        if not context:
            return cls._default_context_folder_template
        return cls._context_to_folder_lookup.get(
            str(context), cls._default_context_folder_template
        )


# clearml/storage/callbacks.py

import logging
import os
from time import time
from typing import Optional, AnyStr, IO
from ..config import config


class ProgressReport(object):
    def __init__(self, verbose, total_size, log, report_chunk_size_mb):
        self.current_status_mb = 0.
        self.last_reported = 0.
        self._tic = time()
        self._verbose = verbose
        self._report_chunk_size = report_chunk_size_mb
        self._log = log
        self._log_flag = False
        self._total_size = total_size

    def __call__(self, chunk_size, *_, **__):
        chunk_size /= 1024. * 1024.
        self.current_status_mb += chunk_size
        last_part = self.current_status_mb - self.last_reported

        if self._verbose or (last_part >= self._report_chunk_size):
            time_diff = time() - self._tic
            self.speed = (last_part / time_diff) if time_diff != 0 else 0
            self._tic = time()
            self.last_reported = self.current_status_mb
            self._report(self._total_size, self.current_status_mb, self.speed)

    def _report(self, total_mb, current_mb, speed_mbps):
        # type: (float, float, float) -> None
        pass


class UploadProgressReport(ProgressReport):
    def __init__(self, filename, verbose, total_size, log, report_chunk_size_mb=0):
        if not report_chunk_size_mb:
            report_chunk_size_mb = int(config.get('storage.log.report_upload_chunk_size_mb', 0) or 5)
        super(UploadProgressReport, self).__init__(verbose, total_size, log, report_chunk_size_mb)
        self._filename = filename

    def _report(self, total_mb, current_mb, speed_mbps):
        # type: (float, float, float) -> None
        self._log.info(
            'Uploading: %.2fMB / %.2fMB @ %.2fMBs from %s' %
            (current_mb, total_mb, speed_mbps, self._filename)
        )

    @classmethod
    def from_stream(cls, stream, filename, verbose, log):
        # type: (IO[AnyStr], str, bool, logging.Logger) -> Optional[UploadProgressReport]
        if hasattr(stream, 'seek'):
            total_size = cls._get_stream_length(stream)
            return UploadProgressReport(filename, verbose, total_size, log)

    @classmethod
    def from_file(cls, filename, verbose, log):
        # type: (str, bool, logging.Logger) -> UploadProgressReport
        total_size_mb = float(os.path.getsize(filename)) / (1024. * 1024.)
        return UploadProgressReport(filename, verbose, total_size_mb, log)

    @staticmethod
    def _get_stream_length(stream):
        # type: (IO[AnyStr]) -> int
        current_position = stream.tell()
        # seek to end of file
        stream.seek(0, 2)
        total_length = stream.tell()
        # seek back to current position to support
        # partially read file-like objects
        stream.seek(current_position or 0)
        return total_length


class DownloadProgressReport(ProgressReport):
    def __init__(self, total_size, verbose, remote_path, log, report_chunk_size_mb=0):
        if not report_chunk_size_mb:
            report_chunk_size_mb = int(config.get('storage.log.report_download_chunk_size_mb', 0) or 5)

        super(DownloadProgressReport, self).__init__(verbose, total_size, log, report_chunk_size_mb)
        self._remote_path = remote_path

    def _report(self, total_mb, current_mb, speed_mbps):
        # type: (float, float, float) -> None
        self._log.info('Downloading: %.2fMB / %.2fMB @ %.2fMBs from %s' %
                       (current_mb, total_mb, speed_mbps, self._remote_path))


# clearml/storage/helper.py

from __future__ import with_statement

import errno
import getpass
import itertools
import json
import os
import platform
import shutil
import sys
import threading
import mimetypes
from abc import ABCMeta, abstractmethod
from collections import namedtuple
from concurrent.futures import ThreadPoolExecutor
from copy import copy
from datetime import datetime
from multiprocessing.pool import ThreadPool
from tempfile import mktemp
from time import time
from types import GeneratorType

import requests
import six
from _socket import gethostname
from attr import attrs, attrib, asdict
from furl import furl
from pathlib2 import Path
from requests import codes as requests_codes
from requests.exceptions import ConnectionError
from six import binary_type, StringIO
from six.moves.queue import Queue, Empty
from six.moves.urllib.parse import urlparse

from .callbacks import UploadProgressReport, DownloadProgressReport
from .util import quote_url
from ..backend_api.session import Session
from ..backend_api.utils import get_http_session_with_retry
from ..backend_config.bucket_config import S3BucketConfigurations, GSBucketConfigurations, AzureContainerConfigurations
from ..config import config, deferred_config
from ..debugging import get_logger
from ..errors import UsageError
from ..utilities.process.mp import ForkSafeRLock, SafeEvent


class StorageError(Exception):
    pass


class DownloadError(Exception):
    pass


@six.add_metaclass(ABCMeta)
class _Driver(object):
    _file_server_hosts = None

    @classmethod
    def get_logger(cls):
        return get_logger('storage')

    @abstractmethod
    def get_container(self, container_name, config=None, **kwargs):
        pass

    @abstractmethod
    def test_upload(self, test_path, config, **kwargs):
        pass

    @abstractmethod
    def upload_object_via_stream(self, iterator, container, object_name, extra, **kwargs):
        pass

    @abstractmethod
    def list_container_objects(self, container, ex_prefix, **kwargs):
        pass

    @abstractmethod
    def get_direct_access(self, remote_path, **kwargs):
        pass

    @abstractmethod
    def download_object(self, obj, local_path, overwrite_existing, delete_on_failure, callback, **kwargs):
        pass

    @abstractmethod
    def download_object_as_stream(self, obj, chunk_size, **kwargs):
        pass

    @abstractmethod
    def delete_object(self, obj, **kwargs):
        pass

    @abstractmethod
    def upload_object(self, file_path, container, object_name, extra, **kwargs):
        pass

    @abstractmethod
    def get_object(self, container_name, object_name, **kwargs):
        pass

    @classmethod
    def get_file_server_hosts(cls):
        if cls._file_server_hosts is None:
            cls._file_server_hosts = [Session.get_files_server_host()] + (Session.legacy_file_servers or [])
        return cls._file_server_hosts


class StorageHelper(object):
    """ Storage helper.
        Used by the entire system to download/upload files.
        Supports both local and remote files (currently local files, network-mapped files, HTTP/S and Amazon S3)
    """
    _temp_download_suffix = '.partially'

    @classmethod
    def _get_logger(cls):
        return get_logger('storage')

    @attrs
    class _PathSubstitutionRule(object):
        registered_prefix = attrib(type=str)
        local_prefix = attrib(type=str)
        replace_windows_sep = attrib(type=bool)
        replace_linux_sep = attrib(type=bool)

        path_substitution_config = 'storage.path_substitution'

        @classmethod
        def load_list_from_config(cls):
            rules_list = []
            for index, sub_config in enumerate(config.get(cls.path_substitution_config, list())):
                rule = cls(
                    registered_prefix=sub_config.get('registered_prefix', None),
                    local_prefix=sub_config.get('local_prefix', None),
                    replace_windows_sep=sub_config.get('replace_windows_sep', False),
                    replace_linux_sep=sub_config.get('replace_linux_sep', False),
                )

                if any(prefix is None for prefix in (rule.registered_prefix, rule.local_prefix)):
                    StorageHelper._get_logger().warning(
                        "Illegal substitution rule configuration '{}[{}]': {}".format(
                            cls.path_substitution_config,
                            index,
                            asdict(rule),
                        ))

                    continue

                if all((rule.replace_windows_sep, rule.replace_linux_sep)):
                    StorageHelper._get_logger().warning(
                        "Only one of replace_windows_sep and replace_linux_sep flags may be set."
                        "'{}[{}]': {}".format(
                            cls.path_substitution_config,
                            index,
                            asdict(rule),
                        ))
                    continue

                rules_list.append(rule)

            return rules_list

    class _UploadData(object):
        @property
        def src_path(self):
            return self._src_path

        @property
        def dest_path(self):
            return self._dest_path

        @property
        def extra(self):
            return self._extra

        @property
        def callback(self):
            return self._callback

        @property
        def retries(self):
            return self._retries

        def __init__(self, src_path, dest_path, extra, callback, retries):
            self._src_path = src_path
            self._dest_path = dest_path
            self._extra = extra
            self._callback = callback
            self._retries = retries

        def __str__(self):
            return "src=%s" % self.src_path

    _helpers = {}  # cache of helper instances

    # global terminate event for async upload threads
    # _terminate = threading.Event()
    _async_upload_threads = set()
    _upload_pool = None
    _upload_pool_pid = None

    # collect all bucket credentials that aren't empty (ignore entries with an empty key or secret)
    _s3_configurations = deferred_config('aws.s3', {}, transform=S3BucketConfigurations.from_config)
    _gs_configurations = deferred_config('google.storage', {}, transform=GSBucketConfigurations.from_config)
    _azure_configurations = deferred_config('azure.storage', {}, transform=AzureContainerConfigurations.from_config)
    _path_substitutions = deferred_config(transform=_PathSubstitutionRule.load_list_from_config)

    @property
    def log(self):
        return self._log

    @property
    def scheme(self):
        return self._scheme

    @property
    def secure(self):
        return self._secure

    @property
    def base_url(self):
        return self._base_url

    @classmethod
    def get(cls, url, logger=None, **kwargs):
        """
        Get a storage helper instance for the given URL

        :return: A StorageHelper instance.
        """

        # Handle URL substitution etc before locating the correct storage driver
        url = cls._canonize_url(url)

        # Get the credentials we should use for this url
        base_url = cls._resolve_base_url(url)

        instance_key = '%s_%s' % (base_url, threading.current_thread().ident or 0)
        # noinspection PyBroadException
        try:
            configs = kwargs.get("configs")
            if configs:
                instance_key += "_{}".format(configs.cache_name)
        except Exception:
            pass

        force_create = kwargs.pop('__force_create', False)
        if (instance_key in cls._helpers) and (not force_create) and base_url != "file://":
            return cls._helpers[instance_key]

        # Don't canonize URL since we already did it
        try:
            instance = cls(base_url=base_url, url=url, logger=logger, canonize_url=False, **kwargs)
        except (StorageError, UsageError) as ex:
            cls._get_logger().error(str(ex))
            return None
        except Exception as ex:
            cls._get_logger().error("Failed creating storage object {} Reason: {}".format(
                base_url or url, ex))
            return None

        cls._helpers[instance_key] = instance
        return instance

    @classmethod
    def get_local_copy(cls, remote_url, skip_zero_size_check=False):
        """
        Download a file from remote URL to a local storage, and return path to local copy,

        :param remote_url: Remote URL. Example: https://example.com/file.jpg s3://bucket/folder/file.mp4 etc.
        :param skip_zero_size_check: If True no error will be raised for files with zero bytes size.
        :return: Path to local copy of the downloaded file. None if error occurred.
        """
        helper = cls.get(remote_url)
        if not helper:
            return None
        # create temp file with the requested file name
        file_name = '.' + remote_url.split('/')[-1].split(os.path.sep)[-1]
        local_path = mktemp(suffix=file_name)
        return helper.download_to_file(remote_url, local_path, skip_zero_size_check=skip_zero_size_check)

    def __init__(
        self,
        base_url,
        url,
        key=None,
        secret=None,
        region=None,
        verbose=False,
        logger=None,
        retries=5,
        token=None,
        **kwargs
    ):
        level = config.get("storage.log.level", None)

        if level:
            try:
                self._get_logger().setLevel(level)
            except (TypeError, ValueError):
                self._get_logger().error('invalid storage log level in configuration: %s' % level)

        self._log = logger or self._get_logger()
        self._verbose = verbose
        self._retries = retries
        self._extra = {}
        self._base_url = base_url
        self._secure = True
        self._driver = None
        self._container = None
        self._conf = None

        if kwargs.get('canonize_url', True):
            url = self._canonize_url(url)

        parsed = urlparse(url)
        self._scheme = parsed.scheme

        if self._scheme == _AzureBlobServiceStorageDriver.scheme:
            self._conf = copy(self._azure_configurations.get_config_by_uri(url))
            if self._conf is None:
                raise StorageError("Missing Azure Blob Storage configuration for {}".format(url))

            if not self._conf.account_name or not self._conf.account_key:
                raise StorageError(
                    "Missing account name or key for Azure Blob Storage access for {}".format(base_url)
                )

            self._driver = _AzureBlobServiceStorageDriver()
            self._container = self._driver.get_container(config=self._conf, account_url=parsed.netloc)

        elif self._scheme == _Boto3Driver.scheme:
            self._conf = copy(self._s3_configurations.get_config_by_uri(url))
            self._secure = self._conf.secure

            final_region = region if region else self._conf.region
            if not final_region:
                final_region = None

            self._conf.update(
                key=key or self._conf.key,
                secret=secret or self._conf.secret,
                multipart=self._conf.multipart,
                region=final_region,
                use_credentials_chain=self._conf.use_credentials_chain,
                token=token or self._conf.token,
                extra_args=self._conf.extra_args,
            )

            if not self._conf.use_credentials_chain:
                if not self._conf.key or not self._conf.secret:
                    raise ValueError(
                        "Missing key and secret for S3 storage access (%s)" % base_url
                    )

            self._driver = _Boto3Driver()
            self._container = self._driver.get_container(
                container_name=self._base_url, retries=retries, config=self._conf)

        elif self._scheme == _GoogleCloudStorageDriver.scheme:
            self._conf = copy(self._gs_configurations.get_config_by_uri(url))
            self._driver = _GoogleCloudStorageDriver()
            self._container = self._driver.get_container(
                container_name=self._base_url,
                config=self._conf
            )

        elif self._scheme in _HttpDriver.schemes:
            self._driver = _HttpDriver(retries=retries)
            self._container = self._driver.get_container(container_name=self._base_url)
        else:  # elif self._scheme == 'file':
            # if this is not a known scheme assume local file
            # url2pathname is specifically intended to operate on (urlparse result).path
            # and returns a cross-platform compatible result
            new_url = normalize_local_path(url)
            self._driver = _FileStorageDriver(new_url)
            # noinspection PyBroadException
            try:
                self._container = self._driver.get_container("")
            except Exception:
                self._container = None

    @classmethod
    def terminate_uploads(cls, force=True, timeout=2.0):
        if force:
            # since async uploaders are daemon threads, we can just return and let them close by themselves
            return
        # signal all threads to terminate and give them a chance for 'timeout' seconds (total, not per-thread)
        # cls._terminate.set()
        remaining_timeout = timeout
        for thread in cls._async_upload_threads:
            t = time()
            # noinspection PyBroadException
            try:
                thread.join(timeout=remaining_timeout)
            except Exception:
                pass
            remaining_timeout -= (time() - t)

    @classmethod
    def get_configuration(cls, bucket_config):
        return cls._s3_configurations.get_config_by_bucket(bucket_config.bucket, bucket_config.host)

    @classmethod
    def add_configuration(cls, bucket_config, log=None, _test_config=True):
        # Try to use existing configuration if we have no key and secret
        use_existing = not bucket_config.is_valid()

        # Get existing config anyway (we'll either try to use it or alert we're replacing it
        existing = cls.get_configuration(bucket_config)

        configs = cls._s3_configurations

        if not use_existing:
            # Test bucket config, fails if unsuccessful
            if _test_config:
                _Boto3Driver._test_bucket_config(bucket_config, log)

            if existing:
                if log:
                    log.warning('Overriding existing configuration for %s/%s'
                                % (existing.host or 'AWS', existing.bucket))
                configs.remove_config(existing)
        else:
            # Try to use existing configuration
            good_config = False
            if existing:
                if log:
                    log.info('Using existing credentials for bucket %s/%s'
                             % (bucket_config.host or 'AWS', bucket_config.bucket))
                good_config = _Boto3Driver._test_bucket_config(existing, log, raise_on_error=False)

            if not good_config:
                # Try to use global key/secret
                configs.update_config_with_defaults(bucket_config)

                if log:
                    log.info('Using global credentials for bucket %s/%s'
                             % (bucket_config.host or 'AWS', bucket_config.bucket))
                if _test_config:
                    _Boto3Driver._test_bucket_config(bucket_config, log)
            else:
                # do not add anything, existing config is OK
                return

        configs.add_config(bucket_config)

    @classmethod
    def add_path_substitution(
            cls,
            registered_prefix,
            local_prefix,
            replace_windows_sep=False,
            replace_linux_sep=False,
    ):
        """
        Add a path substitution rule for storage paths.

        Useful for case where the data was registered under some path, and that
        path was later renamed. This may happen with local storage paths where
        each machine is has different mounts or network drives configurations

        :param registered_prefix: The prefix to search for and replace. This is
            the prefix of the path the data is registered under. This should be the
            exact url prefix, case sensitive, as the data is registered.
        :param local_prefix: The prefix to replace 'registered_prefix' with. This
            is the prefix of the path the data is actually saved under. This should be the
            exact url prefix, case sensitive, as the data is saved under.
        :param replace_windows_sep: If set to True, and the prefix matches, the rest
            of the url has all of the windows path separators (backslash '\') replaced with
            the native os path separator.
        :param replace_linux_sep: If set to True, and the prefix matches, the rest
            of the url has all of the linux/unix path separators (slash '/') replaced with
            the native os path separator.
        """

        if not registered_prefix or not local_prefix:
            raise UsageError("Path substitution prefixes must be non empty strings")

        if replace_windows_sep and replace_linux_sep:
            raise UsageError("Only one of replace_windows_sep and replace_linux_sep may be set.")

        rule = cls._PathSubstitutionRule(
            registered_prefix=registered_prefix,
            local_prefix=local_prefix,
            replace_windows_sep=replace_windows_sep,
            replace_linux_sep=replace_linux_sep,
        )

        cls._path_substitutions.append(rule)

    @classmethod
    def clear_path_substitutions(cls):
        """
        Removes all path substitution rules, including ones from the configuration file.
        """
        cls._path_substitutions = list()

    def get_object_size_bytes(self, remote_url, silence_errors=False):
        # type: (str, bool) -> [int, None]
        """
        Get size of the remote file in bytes.

        :param str remote_url: The url where the file is stored.
            E.g. 's3://bucket/some_file.txt', 'file://local/file'
        :param bool silence_errors: Silence errors that might occur
            when fetching the size of the file. Default: False

        :return: The size of the file in bytes.
            None if the file could not be found or an error occurred.
        """
        size = None
        obj = self.get_object(remote_url, silence_errors=silence_errors)
        if not obj:
            return None
        try:
            if isinstance(self._driver, _HttpDriver) and obj:
                obj = self._driver._get_download_object(obj)  # noqa
                size = obj.headers.get("Content-Length", 0)
            elif hasattr(obj, "size"):
                size = obj.size
                # Google storage has the option to reload the object to get the size
                if size is None and hasattr(obj, "reload"):
                    obj.reload()
                    size = obj.size
            elif hasattr(obj, "content_length"):
                size = obj.content_length
        except (ValueError, AttributeError, KeyError):
            pass
        return size

    def verify_upload(self, folder_uri='', raise_on_error=True, log_on_error=True):
        """
        Verify that this helper can upload files to a folder.

        An upload is possible iff:
            1. the destination folder is under the base uri of the url used to create the helper
            2. the helper has credentials to write to the destination folder

        :param folder_uri: The destination folder to test. Must be an absolute
            url that begins with the base uri of the url used to create the helper.
        :param raise_on_error: Raise an exception if an upload is not possible
        :param log_on_error: Log an error if an upload is not possible
        :return: True, if, and only if, an upload to folder_uri is possible.
        """

        folder_uri = self._canonize_url(folder_uri)

        folder_uri = self.conform_url(folder_uri, self._base_url)

        test_path = self._normalize_object_name(folder_uri)

        if self._scheme == _Boto3Driver.scheme:
            _Boto3Driver._test_bucket_config(
                self._conf,
                self._log,
                test_path=test_path,
                raise_on_error=raise_on_error,
                log_on_error=log_on_error,
            )
        elif self._scheme == _GoogleCloudStorageDriver.scheme:
            self._driver.test_upload(test_path, self._conf)

        elif self._scheme == 'file':
            # Check path exists
            Path(test_path).mkdir(parents=True, exist_ok=True)
            # check path permissions
            Path(test_path).touch(exist_ok=True)

        return folder_uri

    def upload_from_stream(self, stream, dest_path, extra=None, retries=1):
        dest_path = self._canonize_url(dest_path)
        object_name = self._normalize_object_name(dest_path)
        extra = extra.copy() if extra else {}
        extra.update(self._extra)
        last_ex = None
        cb = UploadProgressReport.from_stream(stream, object_name, self._verbose, self._log)
        for i in range(max(1, retries)):
            try:
                self._driver.upload_object_via_stream(
                    iterator=stream,
                    container=self._container,
                    object_name=object_name,
                    callback=cb,
                    extra=extra)
                last_ex = None
                break
            except Exception as ex:
                last_ex = ex
                # seek to beginning if possible
                # noinspection PyBroadException
                try:
                    stream.seek(0)
                except Exception:
                    pass
        if last_ex:
            raise last_ex

        if self.scheme in _HttpDriver.schemes:
            # quote link
            dest_path = quote_url(dest_path)

        return dest_path

    def upload(self, src_path, dest_path=None, extra=None, async_enable=False, cb=None, retries=3):
        if not dest_path:
            dest_path = os.path.basename(src_path)

        dest_path = self._canonize_url(dest_path)
        dest_path = dest_path.replace('\\', '/')

        if cb and self.scheme in _HttpDriver.schemes:
            # store original callback
            a_cb = cb

            # quote link
            def callback(a_path):
                return a_cb(quote_url(a_path) if a_path else a_path)
            # replace callback with wrapper
            cb = callback

        if async_enable:
            data = self._UploadData(src_path=src_path, dest_path=dest_path, extra=extra, callback=cb, retries=retries)
            StorageHelper._initialize_upload_pool()
            return StorageHelper._upload_pool.apply_async(self._do_async_upload, args=(data,))
        else:
            res = self._do_upload(src_path, dest_path, extra, cb, verbose=False, retries=retries)
            if res:
                res = quote_url(res)
            return res

    def list(self, prefix=None):
        """
        List entries in the helper base path.

        Return a list of names inside this helper base path. The base path is
        determined at creation time and is specific for each storage medium.
        For Google Storage and S3 it is the bucket of the path.
        For local files it is the root directory.

        This operation is not supported for http and https protocols.

        :param prefix: If None, return the list as described above. If not, it
            must be a string - the path of a sub directory under the base path.
            the returned list will include only objects under that subdir.

        :return: The paths of all the objects in the storage base
            path under prefix. Listed relative to the base path.

        """

        if prefix:
            if prefix.startswith(self._base_url):
                prefix = prefix[len(self.base_url):].lstrip("/")

            try:
                res = self._driver.list_container_objects(self._container, ex_prefix=prefix)
            except TypeError:
                res = self._driver.list_container_objects(self._container)

            result = [
                obj.name
                for obj in res
                if (obj.name.startswith(prefix) or self._base_url == "file://") and obj.name != prefix
            ]
            if self._base_url == "file://":
                result = [Path(f).as_posix() for f in result]
            return result
        else:
            return [obj.name for obj in self._driver.list_container_objects(self._container)]

    def download_to_file(
            self,
            remote_path,
            local_path,
            overwrite_existing=False,
            delete_on_failure=True,
            verbose=None,
            skip_zero_size_check=False,
            silence_errors=False,
            direct_access=True
    ):
        def next_chunk(astream):
            if isinstance(astream, binary_type):
                chunk = astream
                astream = None
            elif astream:
                try:
                    chunk = next(astream)
                except StopIteration:
                    chunk = None
            else:
                chunk = None
            return chunk, astream

        remote_path = self._canonize_url(remote_path)
        verbose = self._verbose if verbose is None else verbose

        tmp_remote_path = remote_path
        # noinspection PyBroadException
        try:
            tmp_remote_path = normalize_local_path(tmp_remote_path)
            if tmp_remote_path.exists():
                remote_path = "file://{}".format(str(tmp_remote_path))
        except Exception:
            pass
        # Check if driver type supports direct access:
        direct_access_path = self.get_driver_direct_access(remote_path)
        if direct_access_path and direct_access:
            return direct_access_path

        temp_local_path = None
        try:
            if verbose:
                self._log.info('Start downloading from %s' % remote_path)
            if not overwrite_existing and Path(local_path).is_file():
                self._log.warning(
                    'File {} already exists, no need to download, thread id = {}'.format(
                        local_path,
                        threading.current_thread().ident,
                    ),
                )

                return local_path
            if remote_path.startswith("file://"):
                Path(local_path).parent.mkdir(parents=True, exist_ok=True)
                shutil.copyfile(direct_access_path, local_path)
                return local_path
            # we download into temp_local_path so that if we accidentally stop in the middle,
            # we won't think we have the entire file
            temp_local_path = '{}_{}{}'.format(local_path, time(), self._temp_download_suffix)
            obj = self.get_object(remote_path, silence_errors=silence_errors)
            if not obj:
                return None

            # object size in bytes
            total_size_mb = -1
            dl_total_mb = 0.
            download_reported = False
            # chunks size is ignored and always 5Mb
            chunk_size_mb = 5

            # make sure we have the destination folder
            # noinspection PyBroadException
            Path(temp_local_path).parent.mkdir(parents=True, exist_ok=True)

            total_size_bytes = self.get_object_size_bytes(remote_path, silence_errors=silence_errors)
            if total_size_bytes is not None:
                total_size_mb = float(total_size_bytes) / (1024 * 1024)

            # if driver supports download with callback, use it (it might be faster)
            if hasattr(self._driver, 'download_object'):
                # callback
                cb = DownloadProgressReport(total_size_mb, verbose, remote_path, self._log)
                self._driver.download_object(obj, temp_local_path, callback=cb)
                download_reported = bool(cb.last_reported)
                dl_total_mb = cb.current_status_mb
            else:
                stream = self._driver.download_object_as_stream(obj, chunk_size_mb * 1024 * 1024)
                if stream is None:
                    raise ValueError('Could not download %s' % remote_path)
                with open(temp_local_path, 'wb') as fd:
                    data, stream = next_chunk(stream)
                    while data:
                        fd.write(data)
                        data, stream = next_chunk(stream)

            if not skip_zero_size_check and Path(temp_local_path).stat().st_size <= 0:
                raise Exception('downloaded a 0-sized file')

            # if we are on Windows, we need to remove the target file before renaming
            # otherwise posix rename will overwrite the target
            if os.name != 'posix':
                # noinspection PyBroadException
                try:
                    os.remove(local_path)
                except Exception:
                    pass

            # rename temp file to local_file
            # noinspection PyBroadException
            try:
                os.rename(temp_local_path, local_path)
            except Exception:
                # noinspection PyBroadException
                try:
                    os.unlink(temp_local_path)
                except Exception:
                    pass
                # file was downloaded by a parallel process, check we have the final output and delete the partial copy
                path_local_path = Path(local_path)
                if not path_local_path.is_file() or (not skip_zero_size_check and path_local_path.stat().st_size <= 0):
                    raise Exception('Failed renaming partial file, downloaded file exists and a 0-sized file')

            # report download if we are on the second chunk
            if verbose or download_reported:
                self._log.info(
                    'Downloaded %.2f MB successfully from %s , saved to %s' % (dl_total_mb, remote_path, local_path))
            return local_path
        except DownloadError:
            raise
        except Exception as e:
            self._log.error("Could not download {} , err: {} ".format(remote_path, e))
            if delete_on_failure:
                # noinspection PyBroadException
                try:
                    os.remove(temp_local_path)
                except Exception:
                    pass
            return None

    def download_as_stream(self, remote_path, chunk_size=None):
        remote_path = self._canonize_url(remote_path)
        try:
            obj = self.get_object(remote_path)
            return self._driver.download_object_as_stream(
                obj, chunk_size=chunk_size, verbose=self._verbose, log=self.log
            )
        except DownloadError:
            raise
        except Exception as e:
            self._log.error("Could not download file : %s, err:%s " % (remote_path, str(e)))
            return None

    def download_as_nparray(self, remote_path, chunk_size=None):
        try:
            stream = self.download_as_stream(remote_path, chunk_size)
            if stream is None:
                return

            # TODO: ugly py3 hack, please remove ASAP
            if six.PY3 and not isinstance(stream, GeneratorType):
                import numpy as np
                return np.frombuffer(stream, dtype=np.uint8)
            else:
                import numpy as np
                return np.asarray(bytearray(b''.join(stream)), dtype=np.uint8)

        except Exception as e:
            self._log.error("Could not download file : %s, err:%s " % (remote_path, str(e)))

    def delete(self, path):
        return self._driver.delete_object(self.get_object(path))

    def check_write_permissions(self, dest_path=None):
        # create a temporary file, then delete it
        base_url = dest_path or self._base_url
        dest_path = base_url + '/.clearml.test'
        # do not check http/s connection permissions
        if dest_path.startswith('http'):
            return True

        try:
            self.upload_from_stream(stream=six.BytesIO(b'clearml'), dest_path=dest_path)
        except Exception:
            raise ValueError('Insufficient permissions (write failed) for {}'.format(base_url))
        try:
            self.delete(path=dest_path)
        except Exception:
            raise ValueError('Insufficient permissions (delete failed) for {}'.format(base_url))
        return True

    @classmethod
    def download_from_url(cls, remote_url, local_path, overwrite_existing=False, skip_zero_size_check=False):
        """
        Download a file from remote URL to a local storage

        :param remote_url: Remote URL. Example: https://example.com/image.jpg or s3://bucket/folder/file.mp4 etc.
        :param local_path: target location for downloaded file. Example: /tmp/image.jpg
        :param overwrite_existing: If True and local_path exists, it will overwrite it, otherwise print warning
        :param skip_zero_size_check: If True no error will be raised for files with zero bytes size.
        :return: local_path if download was successful.
        """
        helper = cls.get(remote_url)
        if not helper:
            return None
        return helper.download_to_file(
            remote_url, local_path, overwrite_existing=overwrite_existing, skip_zero_size_check=skip_zero_size_check
        )

    def get_driver_direct_access(self, path):
        """
        Check if the helper's driver has a direct access to the file

        :param str path: file path to check access to
        :return: Return the string representation of the file as path if have access to it, else None
        """

        return self._driver.get_direct_access(path)

    @classmethod
    def _canonize_url(cls, url):
        return cls._apply_url_substitutions(url)

    @classmethod
    def _apply_url_substitutions(cls, url):
        def replace_separator(_url, where, sep):
            return _url[:where] + _url[where:].replace(sep, os.sep)

        for index, rule in enumerate(cls._path_substitutions):
            if url.startswith(rule.registered_prefix):
                url = url.replace(
                    rule.registered_prefix,
                    rule.local_prefix,
                    1,  # count. str.replace() does not support keyword arguments
                )

                if rule.replace_windows_sep:
                    url = replace_separator(url, len(rule.local_prefix), '\\')

                if rule.replace_linux_sep:
                    url = replace_separator(url, len(rule.local_prefix), '/')

                break

        return url

    @classmethod
    def _resolve_base_url(cls, base_url):
        parsed = urlparse(base_url)
        if parsed.scheme == _Boto3Driver.scheme:
            conf = cls._s3_configurations.get_config_by_uri(base_url)
            bucket = conf.bucket
            if not bucket:
                parts = Path(parsed.path.strip('/')).parts
                if parts:
                    bucket = parts[0]
            return '/'.join(x for x in ('s3:/', conf.host, bucket) if x)
        elif parsed.scheme == _AzureBlobServiceStorageDriver.scheme:
            conf = cls._azure_configurations.get_config_by_uri(base_url)
            if not conf:
                raise StorageError("Can't find azure configuration for {}".format(base_url))
            return str(furl(base_url).set(path=conf.container_name))
        elif parsed.scheme == _GoogleCloudStorageDriver.scheme:
            conf = cls._gs_configurations.get_config_by_uri(base_url)
            return str(furl(scheme=parsed.scheme, netloc=conf.bucket))
        elif parsed.scheme in _HttpDriver.schemes:
            for files_server in _Driver.get_file_server_hosts():
                if base_url.startswith(files_server):
                    return files_server
            return parsed.scheme + "://"
        else:  # if parsed.scheme == 'file':
            # if we do not know what it is, we assume file
            return 'file://'

    @classmethod
    def conform_url(cls, folder_uri, base_url=None):
        if not folder_uri:
            return folder_uri
        _base_url = cls._resolve_base_url(folder_uri) if not base_url else base_url

        if not folder_uri.startswith(_base_url):
            prev_folder_uri = folder_uri
            if _base_url == 'file://':
                folder_uri = str(Path(folder_uri).absolute())
                if folder_uri.startswith('/'):
                    folder_uri = _base_url + folder_uri
                elif platform.system() == "Windows":
                    folder_uri = ''.join((_base_url, folder_uri))
                else:
                    folder_uri = '/'.join((_base_url, folder_uri))

                cls._get_logger().debug('Upload destination {} amended to {} for registration purposes'.format(
                    prev_folder_uri, folder_uri))
            else:
                raise ValueError('folder_uri: {} does not start with base url: {}'.format(folder_uri, _base_url))

        return folder_uri

    def _absolute_object_name(self, path):
        """ Returns absolute remote path, including any prefix that is handled by the container """
        if not path.startswith(self.base_url):
            return self.base_url.rstrip('/') + '///' + path.lstrip('/')
        return path

    def _normalize_object_name(self, path):
        """ Normalize remote path. Remove any prefix that is already handled by the container """
        if path.startswith(self.base_url):
            path = path[len(self.base_url):]
            if path.startswith('/') and os.name == 'nt':
                path = path[1:]
        if self.scheme in (_Boto3Driver.scheme, _GoogleCloudStorageDriver.scheme,
                           _AzureBlobServiceStorageDriver.scheme):
            path = path.lstrip('/')
        return path

    def _do_async_upload(self, data):
        assert isinstance(data, self._UploadData)
        return self._do_upload(data.src_path, data.dest_path, extra=data.extra, cb=data.callback,
                               verbose=True, retries=data.retries)

    def _upload_from_file(self, local_path, dest_path, extra=None):
        if not hasattr(self._driver, 'upload_object'):
            with open(local_path, 'rb') as stream:
                res = self.upload_from_stream(stream=stream, dest_path=dest_path, extra=extra)
        else:
            object_name = self._normalize_object_name(dest_path)
            extra = extra.copy() if extra else {}
            extra.update(self._extra)
            cb = UploadProgressReport.from_file(local_path, self._verbose, self._log)
            res = self._driver.upload_object(
                file_path=local_path,
                container=self._container,
                object_name=object_name,
                callback=cb,
                extra=extra)
        return res

    def _do_upload(self, src_path, dest_path, extra=None, cb=None, verbose=False, retries=1):
        object_name = self._normalize_object_name(dest_path)
        if cb:
            try:
                cb(None)
            except Exception as e:
                self._log.error("Calling upload callback when starting upload: %s" % str(e))
        if verbose:
            msg = 'Starting upload: {} => {}{}'.format(
                src_path,
                (self._container.name if self._container.name.endswith('/') else self._container.name + '/')
                if self._container and self._container.name else '', object_name)
            if object_name.startswith('file://') or object_name.startswith('/'):
                self._log.debug(msg)
            else:
                self._log.info(msg)
        last_ex = None
        for i in range(max(1, retries)):
            try:
                if not self._upload_from_file(local_path=src_path, dest_path=dest_path, extra=extra):
                    # retry if failed
                    last_ex = ValueError("Upload failed")
                    continue
                last_ex = None
                break
            except Exception as e:
                last_ex = e

        if last_ex:
            self._log.error("Exception encountered while uploading %s" % str(last_ex))
            if cb:
                try:
                    cb(False)
                except Exception as e:
                    self._log.warning("Exception on upload callback: %s" % str(e))
            raise last_ex

        if verbose:
            self._log.debug("Finished upload: %s => %s" % (src_path, object_name))
        if cb:
            try:
                cb(dest_path)
            except Exception as e:
                self._log.warning("Exception on upload callback: %s" % str(e))

        return dest_path

    def get_object(self, path, silence_errors=False):
        # type: (str, bool) -> object
        """
        Gets the remote object stored at path. The data held by the object
        differs depending on where it is stored.

        :param str path: the path where the remote object is stored
        :param bool silence_errors: Silence errors that might occur
            when fetching the remote object

        :return: The remote object
        """
        object_name = self._normalize_object_name(path)
        try:
            return self._driver.get_object(
                container_name=self._container.name if self._container else '', object_name=object_name)
        except ConnectionError:
            raise DownloadError
        except Exception as e:
            if not silence_errors:
                self.log.warning("Storage helper problem for {}: {}".format(str(object_name), str(e)))
            return None

    @staticmethod
    def _initialize_upload_pool():
        if not StorageHelper._upload_pool or StorageHelper._upload_pool_pid != os.getpid():
            StorageHelper._upload_pool_pid = os.getpid()
            StorageHelper._upload_pool = ThreadPool(processes=1)

    @staticmethod
    def close_async_threads():
        if StorageHelper._upload_pool:
            pool = StorageHelper._upload_pool
            StorageHelper._upload_pool = None
            # noinspection PyBroadException
            try:
                pool.terminate()
                pool.join()
            except Exception:
                pass


class _HttpDriver(_Driver):
    """ LibCloud http/https adapter (simple, enough for now) """

    timeout_connection = deferred_config('http.timeout.connection', 30)
    timeout_total = deferred_config('http.timeout.total', 30)
    max_retries = deferred_config('http.download.max_retries', 15)
    min_kbps_speed = 50

    schemes = ('http', 'https')

    class _Container(object):
        _default_backend_session = None

        def __init__(self, name, retries=5, **kwargs):
            self.name = name
            self.session = get_http_session_with_retry(
                total=retries,
                connect=retries,
                read=retries,
                redirect=retries,
                backoff_factor=0.5,
                backoff_max=120,
                status_forcelist=[
                    requests_codes.request_timeout,
                    requests_codes.timeout,
                    requests_codes.bad_gateway,
                    requests_codes.service_unavailable,
                    requests_codes.bandwidth_limit_exceeded,
                    requests_codes.too_many_requests,
                ]
            )
            self.attach_auth_header = any(
                (name.rstrip('/') == host.rstrip('/') or name.startswith(host.rstrip('/') + '/'))
                for host in _HttpDriver.get_file_server_hosts()
            )

        def get_headers(self, _):
            if not self._default_backend_session:
                from ..backend_interface.base import InterfaceBase
                self._default_backend_session = InterfaceBase._get_default_session()

            if self.attach_auth_header:
                return self._default_backend_session.add_auth_headers({})

    class _HttpSessionHandle(object):
        def __init__(self, url, is_stream, container_name, object_name):
            self.url, self.is_stream, self.container_name, self.object_name = \
                url, is_stream, container_name, object_name

    def __init__(self, retries=None):
        self._retries = retries or int(self.max_retries)
        self._containers = {}

    def get_container(self, container_name, config=None, **kwargs):
        if container_name not in self._containers:
            self._containers[container_name] = self._Container(name=container_name, retries=self._retries, **kwargs)
        return self._containers[container_name]

    def upload_object_via_stream(self, iterator, container, object_name, extra=None, callback=None, **kwargs):
        # when sending data in post, there is no connection timeout, just an entire upload timeout
        timeout = int(self.timeout_total)
        url = container.name
        path = object_name
        if not urlparse(url).netloc:
            host, _, path = object_name.partition('/')
            url += host + '/'

        stream_size = 0
        if hasattr(iterator, 'tell') and hasattr(iterator, 'seek'):
            pos = iterator.tell()
            iterator.seek(0, 2)
            stream_size = iterator.tell() - pos
            iterator.seek(pos, 0)
            timeout = max(timeout, (stream_size / 1024) / float(self.min_kbps_speed))
        res = container.session.post(
            url, files={path: iterator}, timeout=timeout, headers=container.get_headers(url)
        )
        if res.status_code != requests.codes.ok:
            raise ValueError('Failed uploading object %s (%d): %s' % (object_name, res.status_code, res.text))

        # call back is useless because we are not calling it while uploading...

        # if callback and stream_size:
        #     try:
        #         callback(stream_size)
        #     except Exception as ex:
        #         log.debug('Exception raised when running callback function: %s' % ex)
        return res

    def list_container_objects(self, *args, **kwargs):
        raise NotImplementedError('List is not implemented for http protocol')

    def delete_object(self, obj, *args, **kwargs):
        assert isinstance(obj, self._HttpSessionHandle)
        container = self._containers[obj.container_name]
        res = container.session.delete(obj.url, headers=container.get_headers(obj.url))
        if res.status_code != requests.codes.ok:
            self.get_logger().warning('Failed deleting object %s (%d): %s' % (
                obj.object_name, res.status_code, res.text))
            return False
        return True

    def get_object(self, container_name, object_name, *args, **kwargs):
        is_stream = kwargs.get('stream', True)
        url = '/'.join((
            container_name[:-1] if container_name.endswith('/') else container_name,
            object_name.lstrip('/')
        ))
        return self._HttpSessionHandle(url, is_stream, container_name, object_name)

    def _get_download_object(self, obj):
        # bypass for session result
        if not isinstance(obj, self._HttpSessionHandle):
            return obj

        container = self._containers[obj.container_name]
        # set stream flag before we send the request
        container.session.stream = obj.is_stream
        res = container.session.get(
            obj.url, timeout=(int(self.timeout_connection), int(self.timeout_total)),
            headers=container.get_headers(obj.url))
        if res.status_code != requests.codes.ok:
            raise ValueError('Failed getting object %s (%d): %s' % (obj.object_name, res.status_code, res.reason))
        return res

    def download_object_as_stream(self, obj, chunk_size=64 * 1024, **_):
        # return iterable object
        obj = self._get_download_object(obj)
        return obj.iter_content(chunk_size=chunk_size)

    def download_object(self, obj, local_path, overwrite_existing=True, delete_on_failure=True, callback=None, **_):
        obj = self._get_download_object(obj)
        p = Path(local_path)
        if not overwrite_existing and p.is_file():
            self.get_logger().warning('failed saving after download: overwrite=False and file exists (%s)' % str(p))
            return
        length = 0
        with p.open(mode='wb') as f:
            for chunk in obj.iter_content(chunk_size=5 * 1024 * 1024):
                # filter out keep-alive new chunks
                if not chunk:
                    continue
                chunk_size = len(chunk)
                f.write(chunk)
                length += chunk_size
                if callback:
                    callback(chunk_size)

        return length

    def get_direct_access(self, remote_path, **_):
        return None

    def test_upload(self, test_path, config, **kwargs):
        return True

    def upload_object(self, file_path, container, object_name, extra, callback=None, **kwargs):
        with open(file_path, 'rb') as stream:
            return self.upload_object_via_stream(iterator=stream, container=container,
                                                 object_name=object_name, extra=extra,  callback=callback, **kwargs)


class _Stream(object):
    encoding = None
    mode = 'rw'
    name = ''
    newlines = '\n'
    softspace = False

    def __init__(self, input_iterator=None):
        self.closed = False
        self._buffer = Queue()
        self._input_iterator = input_iterator
        self._leftover = None

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def close(self):
        self.closed = True

    def flush(self):
        pass

    def fileno(self):
        return 87

    def isatty(self):
        return False

    def next(self):
        while not self.closed or not self._buffer.empty():
            # input stream
            if self._input_iterator:
                try:
                    chunck = next(self._input_iterator)
                    return chunck
                except StopIteration:
                    self.closed = True
                    raise StopIteration()
                except Exception as ex:
                    _Driver.get_logger().error('Failed downloading: %s' % ex)
            else:
                # in/out stream
                try:
                    return self._buffer.get(block=True, timeout=1.)
                except Empty:
                    pass

        raise StopIteration()

    def read(self, size=None):
        try:
            data = self.next() if self._leftover is None else self._leftover
        except StopIteration:
            return six.b('')

        self._leftover = None
        try:
            while size is None or not data or len(data) < size:
                chunk = self.next()
                if chunk is not None:
                    if data is not None:
                        data += chunk
                    else:
                        data = chunk
        except StopIteration:
            pass

        if size is not None and data and len(data) > size:
            self._leftover = data[size:]
            return data[:size]

        return data

    def readline(self, size=None):
        return self.read(size)

    def readlines(self, sizehint=None):
        pass

    def truncate(self, size=None):
        pass

    def write(self, bytes):
        self._buffer.put(bytes, block=True)

    def writelines(self, sequence):
        for s in sequence:
            self.write(s)


class _Boto3Driver(_Driver):
    """ Boto3 storage adapter (simple, enough for now) """

    _min_pool_connections = 512
    _max_multipart_concurrency = deferred_config('aws.boto3.max_multipart_concurrency', 16)
    _pool_connections = deferred_config('aws.boto3.pool_connections', 512)
    _connect_timeout = deferred_config('aws.boto3.connect_timeout', 60)
    _read_timeout = deferred_config('aws.boto3.read_timeout', 60)

    _stream_download_pool_connections = deferred_config('aws.boto3.stream_connections', 128)
    _stream_download_pool = None
    _stream_download_pool_pid = None

    _containers = {}

    scheme = 's3'
    scheme_prefix = str(furl(scheme=scheme, netloc=''))

    _bucket_location_failure_reported = set()

    class _Container(object):
        _creation_lock = ForkSafeRLock()

        def __init__(self, name, cfg):
            try:
                import boto3
                import botocore.client
                from botocore.exceptions import ClientError  # noqa: F401
            except ImportError:
                raise UsageError(
                    'AWS S3 storage driver (boto3) not found. '
                    'Please install driver using: pip install \"boto3>=1.9\"'
                )

            # skip 's3://'
            self.name = name[5:]
            endpoint = (('https://' if cfg.secure else 'http://') + cfg.host) if cfg.host else None

            # boto3 client creation isn't thread-safe (client itself is)
            with self._creation_lock:
                boto_kwargs = {
                    "endpoint_url": endpoint,
                    "use_ssl": cfg.secure,
                    "verify": cfg.verify,
                    "config": botocore.client.Config(
                        max_pool_connections=max(
                            int(_Boto3Driver._min_pool_connections),
                            int(_Boto3Driver._pool_connections)),
                        connect_timeout=int(_Boto3Driver._connect_timeout),
                        read_timeout=int(_Boto3Driver._read_timeout),
                    )
                }
                if not cfg.use_credentials_chain:
                    boto_kwargs["aws_access_key_id"] = cfg.key
                    boto_kwargs["aws_secret_access_key"] = cfg.secret
                    if cfg.token:
                        boto_kwargs["aws_session_token"] = cfg.token

                self.resource = boto3.resource(
                    "s3",
                    **boto_kwargs
                )

                self.config = cfg
                bucket_name = self.name[len(cfg.host) + 1:] if cfg.host else self.name
                self.bucket = self.resource.Bucket(bucket_name)

    @attrs
    class ListResult(object):
        name = attrib(default=None)

    def __init__(self):
        pass

    def _get_stream_download_pool(self):
        if self._stream_download_pool is None or self._stream_download_pool_pid != os.getpid():
            self._stream_download_pool_pid = os.getpid()
            self._stream_download_pool = ThreadPoolExecutor(max_workers=int(self._stream_download_pool_connections))
        return self._stream_download_pool

    def get_container(self, container_name, config=None, **kwargs):
        if container_name not in self._containers:
            self._containers[container_name] = self._Container(name=container_name, cfg=config)
        self._containers[container_name].config.retries = kwargs.get('retries', 5)
        return self._containers[container_name]

    def upload_object_via_stream(self, iterator, container, object_name, callback=None, extra=None, **kwargs):
        import boto3.s3.transfer
        stream = _Stream(iterator)
        try:
            extra_args = {
                'ContentType': get_file_mimetype(object_name),
                **(container.config.extra_args or {})
            }
            container.bucket.upload_fileobj(stream, object_name, Config=boto3.s3.transfer.TransferConfig(
                use_threads=container.config.multipart,
                max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
                num_download_attempts=container.config.retries),
                Callback=callback,
                ExtraArgs=extra_args,
            )
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
            return False
        return True

    def upload_object(self, file_path, container, object_name, callback=None, extra=None, **kwargs):
        import boto3.s3.transfer
        try:
            extra_args = {
                'ContentType': get_file_mimetype(object_name or file_path),
                **(container.config.extra_args or {})
            }
            container.bucket.upload_file(file_path, object_name, Config=boto3.s3.transfer.TransferConfig(
                use_threads=container.config.multipart,
                max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
                num_download_attempts=container.config.retries),
                Callback=callback,
                ExtraArgs=extra_args,
            )
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
            return False
        return True

    def list_container_objects(self, container, ex_prefix=None, **kwargs):
        if ex_prefix:
            res = container.bucket.objects.filter(Prefix=ex_prefix)
        else:
            res = container.bucket.objects.all()
        for res in res:
            yield self.ListResult(name=res.key)

    def delete_object(self, object, **kwargs):
        from botocore.exceptions import ClientError
        object.delete()
        try:
            # Try loading the file to verify deletion
            object.load()
            return False
        except ClientError as e:
            return int(e.response['Error']['Code']) == 404

    def get_object(self, container_name, object_name, *args, **kwargs):
        full_container_name = 's3://' + container_name
        container = self._containers[full_container_name]
        obj = container.resource.Object(container.bucket.name, object_name)
        obj.container_name = full_container_name
        return obj

    def download_object_as_stream(self, obj, chunk_size=64 * 1024, verbose=None, log=None, **_):
        def async_download(a_obj, a_stream, cb, cfg):
            try:
                a_obj.download_fileobj(a_stream, Callback=cb, Config=cfg)
            except Exception as ex:
                (log or self.get_logger()).error('Failed downloading: %s' % ex)
            a_stream.close()

        import boto3.s3.transfer
        # return iterable object
        stream = _Stream()
        container = self._containers[obj.container_name]
        config = boto3.s3.transfer.TransferConfig(
            use_threads=container.config.multipart,
            max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
            num_download_attempts=container.config.retries)
        total_size_mb = obj.content_length / (1024. * 1024.)
        remote_path = os.path.join(obj.container_name, obj.key)
        cb = DownloadProgressReport(total_size_mb, verbose, remote_path, log)
        self._get_stream_download_pool().submit(async_download, obj, stream, cb, config)

        return stream

    def download_object(self, obj, local_path, overwrite_existing=True, delete_on_failure=True, callback=None, **_):
        import boto3.s3.transfer
        p = Path(local_path)
        if not overwrite_existing and p.is_file():
            self.get_logger().warning('failed saving after download: overwrite=False and file exists (%s)' % str(p))
            return
        container = self._containers[obj.container_name]
        Config = boto3.s3.transfer.TransferConfig(
            use_threads=container.config.multipart,
            max_concurrency=self._max_multipart_concurrency if container.config.multipart else 1,
            num_download_attempts=container.config.retries
        )
        obj.download_file(str(p), Callback=callback, Config=Config)

    @classmethod
    def _test_bucket_config(cls, conf, log, test_path='', raise_on_error=True, log_on_error=True):
        try:
            import boto3
            from botocore.exceptions import ClientError
        except ImportError:
            return False

        if not conf.bucket:
            return False
        try:
            if not conf.is_valid():
                raise Exception('Missing credentials')

            fullname = furl(conf.bucket).add(path=test_path).add(path='%s-upload_test' % cls.__module__)
            bucket_name = str(fullname.path.segments[0])
            filename = str(furl(path=fullname.path.segments[1:]))

            data = {
                'user': getpass.getuser(),
                'machine': gethostname(),
                'time': datetime.utcnow().isoformat()
            }

            boto_session = boto3.Session(conf.key, conf.secret, aws_session_token=conf.token)
            endpoint = (('https://' if conf.secure else 'http://') + conf.host) if conf.host else None
            boto_resource = boto_session.resource('s3', region_name=conf.region, endpoint_url=endpoint)
            bucket = boto_resource.Bucket(bucket_name)
            bucket.put_object(Key=filename, Body=six.b(json.dumps(data)))

            region = cls._get_bucket_region(conf=conf, log=log, report_info=True)
            if region and ((conf.region and region != conf.region) or (not conf.region and region != 'us-east-1')):
                msg = "incorrect region specified for bucket %s (detected region %s)" % (conf.bucket, region)
            else:
                return True

        except ClientError as ex:
            msg = ex.response['Error']['Message']
            if log_on_error and log:
                log.error(msg)

            if raise_on_error:
                raise

        except Exception as ex:
            msg = str(ex)
            if log_on_error and log:
                log.error(msg)

            if raise_on_error:
                raise

        msg = ("Failed testing access to bucket %s: " % conf.bucket) + msg

        if log_on_error and log:
            log.error(msg)

        if raise_on_error:
            raise StorageError(msg)

        return False

    @classmethod
    def _get_bucket_region(cls, conf, log=None, report_info=False):
        import boto3
        from botocore.exceptions import ClientError

        if not conf.bucket:
            return None

        def report(msg):
            if log and conf.get_bucket_host() not in cls._bucket_location_failure_reported:
                if report_info:
                    log.debug(msg)
                else:
                    log.warning(msg)
                cls._bucket_location_failure_reported.add(conf.get_bucket_host())

        try:
            boto_session = boto3.Session(conf.key, conf.secret, aws_session_token=conf.token)
            boto_resource = boto_session.resource('s3')
            return boto_resource.meta.client.get_bucket_location(Bucket=conf.bucket)["LocationConstraint"]

        except ClientError as ex:
            report("Failed getting bucket location (region) for bucket "
                   "%s: %s (%s, access_key=%s). Default region will be used. "
                   "This is normal if you do not have GET_BUCKET_LOCATION permission"
                   % (conf.bucket, ex.response['Error']['Message'], ex.response['Error']['Code'], conf.key))
        except Exception as ex:
            report("Failed getting bucket location (region) for bucket %s: %s. Default region will be used."
                   % (conf.bucket, str(ex)))

        return None

    def get_direct_access(self, remote_path, **_):
        return None

    def test_upload(self, test_path, config, **_):
        return True


class _GoogleCloudStorageDriver(_Driver):
    """Storage driver for google cloud storage"""

    _stream_download_pool_connections = deferred_config('google.storage.stream_connections', 128)
    _stream_download_pool = None
    _stream_download_pool_pid = None

    _containers = {}

    scheme = 'gs'
    scheme_prefix = str(furl(scheme=scheme, netloc=''))

    class _Container(object):
        def __init__(self, name, cfg):
            try:
                from google.cloud import storage
                from google.oauth2 import service_account
            except ImportError:
                raise UsageError(
                    'Google cloud driver not found. '
                    'Please install driver using: pip install \"google-cloud-storage>=1.13.2\"'
                )

            self.name = name[len(_GoogleCloudStorageDriver.scheme_prefix):]

            if cfg.credentials_json:
                credentials = service_account.Credentials.from_service_account_file(cfg.credentials_json)
            else:
                credentials = None

            self.client = storage.Client(project=cfg.project, credentials=credentials)
            for adapter in self.client._http.adapters.values():
                if cfg.pool_connections:
                    adapter._pool_connections = cfg.pool_connections
                if cfg.pool_maxsize:
                    adapter._pool_maxsize = cfg.pool_maxsize

            self.config = cfg
            self.bucket = self.client.bucket(self.name)

    def _get_stream_download_pool(self):
        if self._stream_download_pool is None or self._stream_download_pool_pid != os.getpid():
            self._stream_download_pool_pid = os.getpid()
            self._stream_download_pool = ThreadPoolExecutor(max_workers=int(self._stream_download_pool_connections))
        return self._stream_download_pool

    def get_container(self, container_name, config=None, **kwargs):
        if container_name not in self._containers:
            self._containers[container_name] = self._Container(name=container_name, cfg=config)
        self._containers[container_name].config.retries = kwargs.get('retries', 5)
        return self._containers[container_name]

    def upload_object_via_stream(self, iterator, container, object_name, extra=None, **kwargs):
        try:
            blob = container.bucket.blob(object_name)
            blob.upload_from_file(iterator)
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
            return False
        return True

    def upload_object(self, file_path, container, object_name, extra=None, **kwargs):
        try:
            blob = container.bucket.blob(object_name)
            blob.upload_from_filename(file_path)
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
            return False
        return True

    def list_container_objects(self, container, **kwargs):
        return list(container.bucket.list_blobs())

    def delete_object(self, object, **kwargs):
        try:
            object.delete()
        except Exception as ex:
            try:
                from google.cloud.exceptions import NotFound
                if isinstance(ex, NotFound):
                    return False
            except ImportError:
                pass
            name = getattr(object, "name", "")
            self.get_logger().warning("Failed deleting object {}: {}".format(name, ex))
            return False

        return not object.exists()

    def get_object(self, container_name, object_name, *args, **kwargs):
        full_container_name = str(furl(scheme=self.scheme, netloc=container_name))
        container = self._containers[full_container_name]
        obj = container.bucket.blob(object_name)
        obj.container_name = full_container_name
        return obj

    def download_object_as_stream(self, obj, chunk_size=256 * 1024, **_):
        raise NotImplementedError('Unsupported for google storage')

        def async_download(a_obj, a_stream):
            try:
                a_obj.download_to_file(a_stream)
            except Exception as ex:
                self.get_logger().error('Failed downloading: %s' % ex)
            a_stream.close()

        # return iterable object
        stream = _Stream()
        obj.chunk_size = chunk_size
        self._get_stream_download_pool().submit(async_download, obj, stream)

        return stream

    def download_object(self, obj, local_path, overwrite_existing=True, delete_on_failure=True, callback=None, **_):
        p = Path(local_path)
        if not overwrite_existing and p.is_file():
            self.get_logger().warning('failed saving after download: overwrite=False and file exists (%s)' % str(p))
            return
        obj.download_to_filename(str(p))

    def test_upload(self, test_path, config, **_):
        bucket_url = str(furl(scheme=self.scheme, netloc=config.bucket, path=config.subdir))
        bucket = self.get_container(container_name=bucket_url, config=config).bucket

        test_obj = bucket

        if test_path:
            if not test_path.endswith('/'):
                test_path += '/'

            blob = bucket.blob(test_path)

            if blob.exists():
                test_obj = blob

        permissions_to_test = ('storage.objects.get', 'storage.objects.update')
        return set(test_obj.test_iam_permissions(permissions_to_test)) == set(permissions_to_test)

    def get_direct_access(self, remote_path, **_):
        return None


class _AzureBlobServiceStorageDriver(_Driver):
    scheme = 'azure'

    _containers = {}

    class _Container(object):
        def __init__(self, name, config, account_url):
            self.MAX_SINGLE_PUT_SIZE = 4 * 1024 * 1024
            self.SOCKET_TIMEOUT = (300, 2000)
            self.name = name
            self.config = config
            self.account_url = account_url
            try:
                from azure.storage.blob import BlobServiceClient  # noqa

                self.__legacy = False
            except ImportError:
                try:
                    from azure.storage.blob import BlockBlobService  # noqa
                    from azure.common import AzureHttpError  # noqa: F401

                    self.__legacy = True
                except ImportError:
                    raise UsageError(
                        "Azure blob storage driver not found. "
                        "Please install driver using: 'pip install clearml[azure]' or "
                        "pip install '\"azure.storage.blob>=12.0.0\"'"
                    )

            if self.__legacy:
                self.__blob_service = BlockBlobService(
                    account_name=self.config.account_name,
                    account_key=self.config.account_key,
                )
                self.__blob_service.MAX_SINGLE_PUT_SIZE = self.MAX_SINGLE_PUT_SIZE
                self.__blob_service.socket_timeout = self.SOCKET_TIMEOUT
            else:
                credential = {"account_name": self.config.account_name, "account_key": self.config.account_key}
                self.__blob_service = BlobServiceClient(
                    account_url=account_url,
                    credential=credential,
                    max_single_put_size=self.MAX_SINGLE_PUT_SIZE,
                )

        def create_blob_from_data(
            self, container_name, object_name, blob_name, data, max_connections=2,
                progress_callback=None, content_settings=None
        ):
            if self.__legacy:
                self.__blob_service.create_blob_from_bytes(
                    container_name,
                    object_name,
                    data,
                    max_connections=max_connections,
                    progress_callback=progress_callback,
                )
            else:
                client = self.__blob_service.get_blob_client(container_name, blob_name)
                client.upload_blob(
                    data, overwrite=True,
                    max_concurrency=max_connections,
                    content_settings=content_settings,
                )

        def create_blob_from_path(
            self, container_name, blob_name, path, max_connections=2, content_settings=None, progress_callback=None
        ):
            if self.__legacy:
                self.__blob_service.create_blob_from_path(
                    container_name,
                    blob_name,
                    path,
                    max_connections=max_connections,
                    content_settings=content_settings,
                    progress_callback=progress_callback,
                )
            else:
                self.create_blob_from_data(
                    container_name, None, blob_name, open(path, "rb"),
                    max_connections=max_connections,
                    content_settings=content_settings,
                )

        def delete_blob(self, container_name, blob_name):
            if self.__legacy:
                self.__blob_service.delete_blob(
                    container_name,
                    blob_name,
                )
            else:
                client = self.__blob_service.get_blob_client(container_name, blob_name)
                client.delete_blob()

        def exists(self, container_name, blob_name):
            if self.__legacy:
                return not self.__blob_service.exists(container_name, blob_name)
            else:
                client = self.__blob_service.get_blob_client(container_name, blob_name)
                return client.exists()

        def list_blobs(self, container_name, prefix=None):
            if self.__legacy:
                return self.__blob_service.list_blobs(container_name=container_name, prefix=prefix)
            else:
                client = self.__blob_service.get_container_client(container_name)
                return client.list_blobs(name_starts_with=prefix)

        def get_blob_properties(self, container_name, blob_name):
            if self.__legacy:
                return self.__blob_service.get_blob_properties(container_name, blob_name)
            else:
                client = self.__blob_service.get_blob_client(container_name, blob_name)
                return client.get_blob_properties()

        def get_blob_to_bytes(self, container_name, blob_name, progress_callback=None):
            if self.__legacy:
                return self.__blob_service.get_blob_to_bytes(
                    container_name,
                    blob_name,
                    progress_callback=progress_callback,
                )
            else:
                client = self.__blob_service.get_blob_client(container_name, blob_name)
                return client.download_blob().content_as_bytes()

        def get_blob_to_path(self, container_name, blob_name, path, max_connections=10, progress_callback=None):
            if self.__legacy:
                return self.__blob_service.get_blob_to_path(
                    container_name,
                    blob_name,
                    path,
                    max_connections=max_connections,
                    progress_callback=progress_callback,
                )
            else:
                client = self.__blob_service.get_blob_client(container_name, blob_name)
                with open(path, "wb") as file:
                    return client.download_blob(max_concurrency=max_connections).download_to_stream(file)

        def is_legacy(self):
            return self.__legacy

        @property
        def blob_service(self):
            return self.__blob_service

    @attrs
    class _Object(object):
        container = attrib()
        blob_name = attrib()
        content_length = attrib()

    def get_container(self, container_name=None, config=None, account_url=None, **kwargs):
        container_name = container_name or config.container_name
        if container_name not in self._containers:
            self._containers[container_name] = self._Container(
                name=container_name, config=config, account_url=account_url
            )
        # self._containers[container_name].config.retries = kwargs.get('retries', 5)
        return self._containers[container_name]

    def upload_object_via_stream(self, iterator, container, object_name, callback=None, extra=None, **kwargs):
        try:
            from azure.common import AzureHttpError  # noqa
        except ImportError:
            from azure.core.exceptions import HttpResponseError  # noqa

            AzureHttpError = HttpResponseError  # noqa

        blob_name = self._blob_name_from_object_path(object_name, container.name)  # noqa: F841
        try:
            container.create_blob_from_data(
                container.name,
                object_name,
                blob_name,
                iterator.read() if hasattr(iterator, "read") else bytes(iterator),
                max_connections=2,
                progress_callback=callback,
                )
            return True
        except AzureHttpError as ex:
            self.get_logger().error('Failed uploading (Azure error): %s' % ex)
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
        return False

    def upload_object(self, file_path, container, object_name, callback=None, extra=None, **kwargs):
        try:
            from azure.common import AzureHttpError  # noqa
        except ImportError:
            from azure.core.exceptions import HttpResponseError  # noqa

            AzureHttpError = HttpResponseError  # noqa

        blob_name = self._blob_name_from_object_path(object_name, container.name)
        stream = None
        try:
            from azure.storage.blob import ContentSettings  # noqa

            container.create_blob_from_path(
                container.name,
                blob_name,
                file_path,
                max_connections=2,
                content_settings=ContentSettings(content_type=get_file_mimetype(object_name or file_path)),
                progress_callback=callback,
            )
            return True
        except AzureHttpError as ex:
            self.get_logger().error('Failed uploading (Azure error): %s' % ex)
        except Exception as ex:
            self.get_logger().error('Failed uploading: %s' % ex)
        finally:
            if stream:
                stream.close()

    def list_container_objects(self, container, ex_prefix=None, **kwargs):
        return list(container.list_blobs(container_name=container.name, prefix=ex_prefix))

    def delete_object(self, object, **kwargs):
        container = object.container
        container.delete_blob(
            container.name,
            object.blob_name,
        )
        return not object.container.exists(container.name, object.blob_name)

    def get_object(self, container_name, object_name, *args, **kwargs):
        container = self._containers.get(container_name)
        if not container:
            raise StorageError("Container `{}` not found for object {}".format(container_name, object_name))

        # blob_name = self._blob_name_from_object_path(object_name, container_name)
        blob = container.get_blob_properties(container.name, object_name)

        if container.is_legacy():
            return self._Object(container=container, blob_name=blob.name, content_length=blob.properties.content_length)
        else:
            return self._Object(container=container, blob_name=blob.name, content_length=blob.size)

    def download_object_as_stream(self, obj, verbose, *_, **__):
        container = obj.container
        total_size_mb = obj.content_length / (1024. * 1024.)
        remote_path = os.path.join(
            "{}://".format(self.scheme), container.config.account_name, container.name, obj.blob_name
        )
        cb = DownloadProgressReport(total_size_mb, verbose, remote_path, self.get_logger())
        blob = container.get_blob_to_bytes(
            container.name,
            obj.blob_name,
            progress_callback=cb,
        )
        if container.is_legacy():
            return blob.content
        else:
            return blob

    def download_object(self, obj, local_path, overwrite_existing=True, delete_on_failure=True, callback=None, **_):
        p = Path(local_path)
        if not overwrite_existing and p.is_file():
            self.get_logger().warning("failed saving after download: overwrite=False and file exists (%s)" % str(p))
            return

        download_done = SafeEvent()
        download_done.counter = 0

        def callback_func(current, total):
            if callback:
                chunk = current - download_done.counter
                download_done.counter += chunk
                callback(chunk)
            if current >= total:
                download_done.set()

        container = obj.container
        container.blob_service.MAX_SINGLE_GET_SIZE = 5 * 1024 * 1024
        _ = container.get_blob_to_path(
            container.name,
            obj.blob_name,
            local_path,
            max_connections=10,
            progress_callback=callback_func,
        )
        if container.is_legacy():
            download_done.wait()

    def test_upload(self, test_path, config, **_):
        container = self.get_container(config=config)
        try:
            container.blob_service.get_container_properties(container.name)
        except Exception:
            return False
        else:
            # Using the account Key, we can always upload...
            return True

    @classmethod
    def _blob_name_from_object_path(cls, name, container_name):
        scheme = urlparse(name).scheme
        if scheme:
            if scheme != cls.scheme:
                raise StorageError(
                    "When using a URL, only the `{}` scheme is supported for Azure storage: {}",
                    cls.scheme,
                    name,
                )

            f = furl(name)

            if not f.path.segments:
                raise StorageError(
                    "Missing container name in URL {}",
                    name,
                )

            parsed_container_name = f.path.segments[0]

            if parsed_container_name != container_name:
                raise StorageError(
                    "Container name mismatch (expected {}, found {}) in {}",
                    container_name,
                    parsed_container_name,
                    name,
                )

            if len(f.path.segments) == 1:
                raise StorageError(
                    "No path found following container name {} in {}",
                    container_name,
                    name,
                )

            return f.path.segments[0], os.path.join(*f.path.segments[1:])

        return name

    def get_direct_access(self, remote_path, **_):
        return None


class _FileStorageDriver(_Driver):
    """
    A base StorageDriver to derive from.
    """

    scheme = "file"
    CHUNK_SIZE = 8096
    IGNORE_FOLDERS = ['.lock', '.hash']
    Object = namedtuple("Object", ['name', 'size', 'extra', 'driver', 'container', 'hash', 'meta_data'])

    class _Container(object):
        def __init__(self, name, extra, driver):
            self.name = name
            self.extra = extra
            self.driver = driver

    def __init__(self, key, secret=None, secure=True, host=None, port=None,
                 **kwargs):

        # Use the key as the path to the storage
        self.base_path = key

    def _make_path(self, path, ignore_existing=True):
        """
        Create a path by checking if it already exists
        """

        try:
            os.makedirs(path)
        except OSError:
            exp = sys.exc_info()[1]
            if exp.errno == errno.EEXIST and not ignore_existing:
                raise exp

    def _check_container_name(self, container_name):
        """
        Check if the container name is valid

        :param container_name: Container name
        :type container_name: ``str``
        """

        if '/' in container_name or '\\' in container_name:
            raise ValueError("Container name \"{}\" cannot contain \\ or / ".format(container_name))

    def _make_container(self, container_name):
        """
        Create a container instance

        :param container_name: Container name.
        :type container_name: ``str``

        :return: A Container instance.
        """
        container_name = container_name or '.'
        self._check_container_name(container_name)

        full_path = os.path.realpath(os.path.join(self.base_path, container_name))

        try:
            stat = os.stat(full_path)
            if not os.path.isdir(full_path):
                raise OSError("Target path \"{}\" is not a directory".format(full_path))
        except OSError:
            raise OSError("Target path \"{}\" is not accessible or does not exist".format(full_path))

        extra = {
            'creation_time': stat.st_ctime,
            'access_time': stat.st_atime,
            'modify_time': stat.st_mtime,
        }

        return self._Container(name=container_name, extra=extra, driver=self)

    def _make_object(self, container, object_name):
        """
        Create an object instance

        :param container: Container.
        :type container: :class:`Container`

        :param object_name: Object name.
        :type object_name: ``str``

        :return: A Object instance.
        """

        full_path = os.path.realpath(os.path.join(self.base_path, container.name if container else '.', object_name))

        if os.path.isdir(full_path):
            raise ValueError("Target path \"{}\" already exist".format(full_path))

        try:
            stat = os.stat(full_path)
        except Exception:
            raise ValueError("Cannot access target path \"{}\"".format(full_path))

        extra = {
            'creation_time': stat.st_ctime,
            'access_time': stat.st_atime,
            'modify_time': stat.st_mtime,
        }

        return self.Object(name=object_name, size=stat.st_size, extra=extra,
                           driver=self, container=container, hash=None, meta_data=None)

    def iterate_containers(self):
        """
        Return a generator of containers.

        :return: A generator of Container instances.
        """

        for container_name in os.listdir(self.base_path):
            full_path = os.path.join(self.base_path, container_name)
            if not os.path.isdir(full_path):
                continue
            yield self._make_container(container_name)

    def _get_objects(self, container):
        """
        Recursively iterate through the file-system and return the object names
        """

        cpath = self.get_container_cdn_url(container, check=True)

        for folder, subfolders, files in os.walk(cpath, topdown=True):
            # Remove unwanted subfolders
            for subf in self.IGNORE_FOLDERS:
                if subf in subfolders:
                    subfolders.remove(subf)

            for name in files:
                full_path = os.path.join(folder, name)
                object_name = os.path.relpath(full_path, start=cpath)
                yield self._make_object(container, object_name)

    def iterate_container_objects(self, container):
        """
        Returns a generator of objects for the given container.

        :param container: Container instance
        :type container: :class:`Container`

        :return: A generator of Object instances.
        """

        return self._get_objects(container)

    def get_container(self, container_name, **_):
        """
        Return a container instance.

        :param container_name: Container name.
        :type container_name: ``str``

        :return: A Container instance.
        """
        return self._make_container(container_name)

    def get_container_cdn_url(self, container, check=False):
        """
        Return a container CDN URL.

        :param container: Container instance
        :type  container: :class:`Container`

        :param check: Indicates if the path's existence must be checked
        :type check: ``bool``

        :return: A CDN URL for this container.
        """
        path = os.path.realpath(os.path.join(self.base_path, container.name if container else '.'))

        if check and not os.path.isdir(path):
            raise ValueError("Target path \"{}\" does not exist".format(path))

        return path

    def get_object(self, container_name, object_name, **_):
        """
        Return an object instance.

        :param container_name: Container name.
        :type  container_name: ``str``

        :param object_name: Object name.
        :type  object_name: ``str``

        :return: An Object instance.
        """
        container = self._make_container(container_name)
        if os.path.isfile(os.path.join(container_name, object_name)):
            return self.Object(
                name=object_name,
                container=container,
                size=Path(object_name).stat().st_size,
                driver=self,
                extra=None,
                hash=None,
                meta_data=None,
            )
        return self._make_object(container, object_name)

    def get_object_cdn_url(self, obj):
        """
        Return an object CDN URL.

        :param obj: Object instance
        :type  obj: :class:`Object`

        :return: A CDN URL for this object.
        """
        return os.path.realpath(os.path.join(self.base_path, obj.container.name, obj.name))

    def download_object(self, obj, destination_path, overwrite_existing=False, delete_on_failure=True, **_):
        """
        Download an object to the specified destination path.

        :param obj: Object instance.
        :type obj: :class:`Object`

        :param destination_path: Full path to a file or a directory where the
                                incoming file will be saved.
        :type destination_path: ``str``

        :param overwrite_existing: True to overwrite an existing file,
            defaults to False.
        :type overwrite_existing: ``bool``

        :param delete_on_failure: True to delete a partially downloaded file if
        the download was not successful (hash mismatch / file size).
        :type delete_on_failure: ``bool``

        :return: True, if an object has been successfully downloaded, False, otherwise.
        """

        obj_path = self.get_object_cdn_url(obj)
        base_name = os.path.basename(destination_path)

        if not base_name and not os.path.exists(destination_path):
            raise ValueError('Path \"{}\" does not exist'.format(destination_path))

        if not base_name:
            file_path = os.path.join(destination_path, obj.name)
        else:
            file_path = destination_path

        if os.path.exists(file_path) and not overwrite_existing:
            raise ValueError('File \"{}\" already exists, but overwrite_existing=False'.format(file_path))

        try:
            shutil.copy(obj_path, file_path)
        except IOError:
            if delete_on_failure:
                # noinspection PyBroadException
                try:
                    os.unlink(file_path)
                except Exception:
                    pass
            return False

        return True

    def download_object_as_stream(self, obj, chunk_size=None, **_):
        """
        Return a generator which yields object data.

        :param obj: Object instance
        :type obj: :class:`Object`

        :param chunk_size: Optional chunk size (in bytes).
        :type chunk_size: ``int``

        :return: A stream of binary chunks of data.
        """
        path = self.get_object_cdn_url(obj)
        with open(path, 'rb') as obj_file:
            for data in self._read_in_chunks(obj_file, chunk_size=chunk_size):
                yield data

    def upload_object(self, file_path, container, object_name, extra=None, verify_hash=True, **_):
        """
        Upload an object currently located on a disk.

        :param file_path: Path to the object on disk.
        :type file_path: ``str``

        :param container: Destination container.
        :type container: :class:`Container`

        :param object_name: Object name.
        :type object_name: ``str``

        :param verify_hash: Verify hast
        :type verify_hash: ``bool``

        :param extra: (optional) Extra attributes (driver specific).
        :type extra: ``dict``
        """
        path = self.get_container_cdn_url(container, check=False)
        obj_path = os.path.join(path, object_name)
        base_path = os.path.dirname(obj_path)

        self._make_path(base_path)

        shutil.copy(file_path, obj_path)

        os.chmod(obj_path, int('664', 8))

        return self._make_object(container, object_name)

    def upload_object_via_stream(self, iterator, container, object_name, extra=None, **kwargs):
        """
        Upload an object using an iterator.

        If a provider supports it, chunked transfer encoding is used and you
        don't need to know in advance the amount of data to be uploaded.

        Otherwise if a provider doesn't support it, iterator will be exhausted
        so a total size for data to be uploaded can be determined.

        Note: Exhausting the iterator means that the whole data must be
        buffered in memory which might result in memory exhausting when
        uploading a very large object.

        If a file is located on a disk you are advised to use upload_object
        function which uses fs.stat function to determine the file size and it
        doesn't need to buffer whole object in the memory.

        :type iterator: ``object``
        :param iterator: An object which implements the iterator
                         interface and yields binary chunks of data.

        :type container: :class:`Container`
        :param container: Destination container.

        :type object_name: ``str``
        :param object_name: Object name.

        :type extra: ``dict``
        :param extra: (optional) Extra attributes (driver specific). Note:
            This dictionary must contain a 'content_type' key which represents
            a content type of the stored object.
        """
        path = self.get_container_cdn_url(container, check=True)
        obj_path = os.path.join(path, object_name)
        base_path = os.path.dirname(obj_path)
        self._make_path(base_path)

        obj_path = os.path.realpath(obj_path)
        with open(obj_path, 'wb' if not isinstance(iterator, StringIO) else 'wt') as obj_file:
            obj_file.write(iterator.read() if hasattr(iterator, 'read') else bytes(iterator))

        os.chmod(obj_path, int('664', 8))
        return self._make_object(container, object_name)

    def delete_object(self, obj, **_):
        """
        Delete an object.

        :type obj: :class:`Object`
        :param obj: Object instance.

        :return: True on success.
        """
        if not obj:
            return False

        path = self.get_object_cdn_url(obj)

        try:
            os.unlink(path)
        except Exception:
            return False

        # # Check and delete all the empty parent folders
        # path = os.path.dirname(path)
        # container_url = obj.container.get_cdn_url()
        #
        # # Delete the empty parent folders till the container's level
        # while path != container_url:
        #     try:
        #         os.rmdir(path)
        #     except OSError:
        #         exp = sys.exc_info()[1]
        #         if exp.errno == errno.ENOTEMPTY:
        #             break
        #         raise exp
        #
        #     path = os.path.dirname(path)

        return True

    def create_container(self, container_name):
        """
        Create a new container.

        :type container_name: ``str``
        :param container_name: Container name.

        :return: A Container instance on success.
        """
        container_name = container_name or '.'
        self._check_container_name(container_name)

        path = os.path.join(self.base_path, container_name)

        try:
            self._make_path(path, ignore_existing=False)
        except OSError:
            exp = sys.exc_info()[1]
            if exp.errno == errno.EEXIST:
                raise ValueError('Container \"{}\" with this name already exists. The name '
                                 'must be unique among all the containers in the '
                                 'system'.format(container_name))
            else:
                raise ValueError('Error creating container \"{}\"'.format(container_name))
        except Exception:
            raise ValueError('Error creating container \"{}\"'.format(container_name))

        return self._make_container(container_name)

    def delete_container(self, container):
        """
        Delete a container.

        :type container: :class:`Container`
        :param container: Container instance

        :return: True on success, False otherwise.
        """

        # Check if there are any objects inside this
        for obj in self._get_objects(container):
            raise ValueError('Container \"{}\" is not empty'.format(container.name))

        path = self.get_container_cdn_url(container, check=True)

        # noinspection PyBroadException
        try:
            shutil.rmtree(path)
        except Exception:
            return False

        return True

    def list_container_objects(self, container, **kwargs):
        return list(self.iterate_container_objects(container))

    @staticmethod
    def _read_in_chunks(iterator, chunk_size=None, fill_size=False, yield_empty=False):
        """
        Return a generator which yields data in chunks.

        :param iterator: An object which implements an iterator interface
                         or a File like object with read method.
        :type iterator: :class:`object` which implements iterator interface.

        :param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
        :type chunk_size: ``int``

        :param fill_size: If True, make sure chunks are exactly chunk_size in
                          length (except for last chunk).
        :type fill_size: ``bool``

        :param yield_empty: If true and iterator returned no data, only yield empty
                            bytes object
        :type yield_empty: ``bool``

        TODO: At some point in the future we could use byte arrays here if version
        >= Python 3. This should speed things up a bit and reduce memory usage.
        """
        chunk_size = chunk_size or _FileStorageDriver.CHUNK_SIZE
        if six.PY3:
            from io import FileIO as file

        if isinstance(iterator, (file)):
            get_data = iterator.read
            args = (chunk_size,)
        else:
            get_data = next
            args = (iterator,)

        data = bytes('')
        empty = False

        while not empty or len(data) > 0:
            if not empty:
                try:
                    chunk = bytes(get_data(*args))
                    if len(chunk) > 0:
                        data += chunk
                    else:
                        empty = True
                except StopIteration:
                    empty = True

            if len(data) == 0:
                if empty and yield_empty:
                    yield bytes('')

                return

            if fill_size:
                if empty or len(data) >= chunk_size:
                    yield data[:chunk_size]
                    data = data[chunk_size:]
            else:
                yield data
                data = bytes('')

    def get_direct_access(self, remote_path, **_):
        # this will always make sure we have full path and file:// prefix
        full_url = StorageHelper.conform_url(remote_path)
        # now get rid of the file:// prefix
        path = Path(full_url[7:])
        if not path.exists():
            raise ValueError("Requested path does not exist: {}".format(path))
        return path.as_posix()

    def test_upload(self, test_path, config, **kwargs):
        return True


def normalize_local_path(local_path):
    """
    Get a normalized local path

    :param local_path: Path of the local file/dir
    :type local_path: str

    :return: Normalized local path
    :rtype: Path
    """
    local_path = os.path.normpath(local_path)
    local_path = os.path.expanduser(local_path)
    local_path = os.path.expandvars(local_path)
    local_path = os.path.realpath(local_path)
    local_path = os.path.abspath(local_path)
    local_path = Path(local_path)
    return local_path


def get_file_mimetype(file_path):
    """
    Get MIME types of a file

    :param file_path: Path of the local file
    :type file_path: str

    :return: File MIME type. Return None if failed to get it
    :rtype: str
    """
    # noinspection PyBroadException
    try:
        file_path = Path(file_path).resolve()
        mimetype, _ = mimetypes.guess_type(file_path.as_posix())
        if mimetype:
            return mimetype
    except Exception:
        pass
    return 'binary/octet-stream'


driver_schemes = set(
    filter(
        None,
        itertools.chain(
            (getattr(cls, "scheme", None) for cls in _Driver.__subclasses__()),
            *(getattr(cls, "schemes", []) for cls in _Driver.__subclasses__())
        )
    )
)

remote_driver_schemes = driver_schemes - {_FileStorageDriver.scheme}


# clearml/storage/manager.py

import fnmatch
import os
import shutil
import tarfile
from multiprocessing.pool import ThreadPool
from random import random
from time import time
from typing import List, Optional
from zipfile import ZipFile
from six.moves.urllib.parse import urlparse

from pathlib2 import Path

from .cache import CacheManager
from .helper import StorageHelper
from .util import encode_string_to_filename
from ..debugging.log import LoggerRoot


class StorageManager(object):
    """
    StorageManager is helper interface for downloading & uploading files to supported remote storage
    Support remote servers: http(s)/S3/GS/Azure/File-System-Folder
    Cache is enabled by default for all downloaded remote urls/files
    """

    @classmethod
    def get_local_copy(
        cls, remote_url, cache_context=None, extract_archive=True, name=None, force_download=False
    ):
        # type: (str, Optional[str], bool, Optional[str], bool) -> [str, None]
        """
        Get a local copy of the remote file. If the remote URL is a direct file access,
        the returned link is the same, otherwise a link to a local copy of the url file is returned.
        Caching is enabled by default, cache limited by number of stored files per cache context.
        Oldest accessed files are deleted when cache is full.

        :param str remote_url: remote url link (string)
        :param str cache_context: Optional caching context identifier (string), default context 'global'
        :param bool extract_archive: if True returned path will be a cached folder containing the archive's content,
            currently only zip files are supported.
        :param str name: name of the target file
        :param bool force_download: download file from remote even if exists in local cache
        :return: Full path to local copy of the requested url. Return None on Error.
        """
        cache = CacheManager.get_cache_manager(cache_context=cache_context)
        cached_file = cache.get_local_copy(remote_url=remote_url, force_download=force_download)
        if extract_archive and cached_file:
            # this will get us the actual cache (even with direct access)
            cache_path_encoding = Path(cache.get_cache_folder()) / cache.get_hashed_url_file(remote_url)
            return cls._extract_to_cache(
                cached_file, name, cache_context, cache_path_encoding=cache_path_encoding.as_posix())
        return cached_file

    @classmethod
    def upload_file(
        cls, local_file, remote_url, wait_for_upload=True, retries=3
    ):  # type: (str, str, bool, int) -> str
        """
        Upload a local file to a remote location. remote url is the finale destination of the uploaded file.

        Examples:

        .. code-block:: py

            upload_file('/tmp/artifact.yaml', 'http://localhost:8081/manual_artifacts/my_artifact.yaml')
            upload_file('/tmp/artifact.yaml', 's3://a_bucket/artifacts/my_artifact.yaml')
            upload_file('/tmp/artifact.yaml', '/mnt/share/folder/artifacts/my_artifact.yaml')

        :param str local_file: Full path of a local file to be uploaded
        :param str remote_url: Full path or remote url to upload to (including file name)
        :param bool wait_for_upload: If False, return immediately and upload in the background. Default True.
        :param int retries: Number of retries before failing to upload file, default 3.
        :return: Newly uploaded remote URL.
        """
        return CacheManager.get_cache_manager().upload_file(
            local_file=local_file,
            remote_url=remote_url,
            wait_for_upload=wait_for_upload,
            retries=retries,
        )

    @classmethod
    def set_cache_file_limit(
        cls, cache_file_limit, cache_context=None
    ):  # type: (int, Optional[str]) -> int
        """
        Set the cache context file limit. File limit is the maximum number of files the specific cache context holds.
        Notice, there is no limit on the size of these files, only the total number of cached files.

        :param int cache_file_limit: New maximum number of cached files
        :param str cache_context: Optional cache context identifier, default global context
        :return: The new cache context file limit.
        """
        return CacheManager.get_cache_manager(
            cache_context=cache_context, cache_file_limit=cache_file_limit
        ).set_cache_limit(cache_file_limit)

    @classmethod
    def _extract_to_cache(
            cls,
            cached_file,  # type: str
            name,  # type: str
            cache_context=None,  # type: Optional[str]
            target_folder=None,  # type: Optional[str]
            cache_path_encoding=None,  # type: Optional[str]
            force=False,  # type: bool
    ):
        # type: (...) -> str
        """
        Extract cached file to cache folder
        :param str cached_file: local copy of archive file
        :param str name: name of the target file
        :param str cache_context: cache context id
        :param str target_folder: specify target path to use for archive extraction
        :param str cache_path_encoding: specify representation of the local path of the cached files,
            this will always point to local cache folder, even if we have direct access file.
            Used for extracting the cached archived based on cache_path_encoding
        :param bool force: Force archive extraction even if target folder exists
        :return: cached folder containing the extracted archive content
        """
        if not cached_file:
            return cached_file

        cached_file = Path(cached_file)
        cache_path_encoding = Path(cache_path_encoding) if cache_path_encoding else None

        # we support zip and tar.gz files auto-extraction
        suffix = cached_file.suffix.lower()
        if suffix == '.gz':
            suffix = ''.join(a.lower() for a in cached_file.suffixes[-2:])

        if suffix not in (".zip", ".tgz", ".tar.gz"):
            return str(cached_file)

        cache_folder = Path(cache_path_encoding or cached_file).parent
        archive_suffix = (cache_path_encoding or cached_file).name[:-len(suffix)]
        name = encode_string_to_filename(name) if name else name
        if target_folder:
            target_folder = Path(target_folder)
        else:
            target_folder = cache_folder / CacheManager.get_context_folder_lookup(
                cache_context).format(archive_suffix, name)

        if target_folder.is_dir() and not force:
            # noinspection PyBroadException
            try:
                target_folder.touch(exist_ok=True)
                return target_folder.as_posix()
            except Exception:
                pass

        base_logger = LoggerRoot.get_base_logger()
        try:
            # if target folder exists, meaning this is forced ao we extract directly into target folder
            if target_folder.is_dir():
                temp_target_folder = target_folder
            else:
                temp_target_folder = cache_folder / "{0}_{1}_{2}".format(
                    target_folder.name, time() * 1000, str(random()).replace('.', ''))
                temp_target_folder.mkdir(parents=True, exist_ok=True)

            if suffix == ".zip":
                ZipFile(cached_file.as_posix()).extractall(path=temp_target_folder.as_posix())
            elif suffix == ".tar.gz":
                with tarfile.open(cached_file.as_posix()) as file:
                    file.extractall(temp_target_folder.as_posix())
            elif suffix == ".tgz":
                with tarfile.open(cached_file.as_posix(), mode='r:gz') as file:
                    file.extractall(temp_target_folder.as_posix())

            if temp_target_folder != target_folder:
                # we assume we will have such folder if we already extract the file
                # noinspection PyBroadException
                try:
                    # if rename fails, it means that someone else already manged to extract the file, delete the current
                    # folder and return the already existing cached zip folder
                    shutil.move(temp_target_folder.as_posix(), target_folder.as_posix())
                except Exception:
                    if target_folder.exists():
                        target_folder.touch(exist_ok=True)
                    else:
                        base_logger.warning(
                            "Failed renaming {0} to {1}".format(temp_target_folder.as_posix(), target_folder.as_posix()))
                    try:
                        shutil.rmtree(temp_target_folder.as_posix())
                    except Exception as ex:
                        base_logger.warning(
                            "Exception {}\nFailed deleting folder {}".format(ex, temp_target_folder.as_posix()))
        except Exception as ex:
            # failed extracting the file:
            base_logger.warning(
                "Exception {}\nFailed extracting zip file {}".format(ex, cached_file.as_posix()))
            # noinspection PyBroadException
            try:
                target_folder.rmdir()
            except Exception:
                pass
            return cached_file.as_posix()
        return target_folder.as_posix()

    @classmethod
    def get_files_server(cls):
        from ..backend_api import Session
        return Session.get_files_server_host()

    @classmethod
    def upload_folder(cls, local_folder, remote_url, match_wildcard=None):
        # type: (str, str, Optional[str]) -> Optional[str]
        """
        Upload local folder recursively to a remote storage, maintaining the sub folder structure
        in the remote storage.

        .. note::

            If we have a local file `~/folder/sub/file.ext` then
            `StorageManager.upload_folder('~/folder/', 's3://bucket/')`
            will create `s3://bucket/sub/file.ext`

        :param str local_folder: Local folder to recursively upload
        :param str remote_url: Target remote storage location, tree structure of `local_folder` will
            be created under the target remote_url. Supports Http/S3/GS/Azure and shared filesystem.
            Example: 's3://bucket/data/'
        :param str match_wildcard: If specified only upload files matching the `match_wildcard`
            Example: `*.json`
            Notice: target file size/date are not checked. Default True, always upload.
            Notice if uploading to http, we will always overwrite the target.
        :return: Newly uploaded remote URL or None on error.
        """

        base_logger = LoggerRoot.get_base_logger()

        if not Path(local_folder).is_dir():
            base_logger.error("Local folder '{}' does not exist".format(local_folder))
            return
        local_folder = str(Path(local_folder))
        results = []
        helper = StorageHelper.get(remote_url)
        with ThreadPool() as pool:
            for path in Path(local_folder).rglob(match_wildcard or "*"):
                if not path.is_file():
                    continue
                results.append(
                    pool.apply_async(
                        helper.upload,
                        args=(str(path), str(path).replace(local_folder, remote_url)),
                    )
                )

            success = 0
            failed = 0
            for res in results:
                # noinspection PyBroadException
                try:
                    res.get()  # Reraise the exceptions from remote call (if any)
                    success += 1
                except Exception:
                    failed += 1

            if failed == 0:
                return remote_url

            base_logger.error("Failed uploading {}/{} files from {}".format(failed, success + failed, local_folder))

    @classmethod
    def download_file(
        cls, remote_url, local_folder=None, overwrite=False, skip_zero_size_check=False, silence_errors=False
    ):
        # type: (str, Optional[str], bool, bool, bool) -> Optional[str]
        """
        Download remote file to the local machine, maintaining the sub folder structure from the
        remote storage.

        .. note::

            If we have a remote file `s3://bucket/sub/file.ext` then
            `StorageManager.download_file('s3://bucket/sub/file.ext', '~/folder/')`
            will create `~/folder/sub/file.ext`
        :param str remote_url: Source remote storage location, path of `remote_url` will
            be created under the target local_folder. Supports S3/GS/Azure and shared filesystem.
            Example: 's3://bucket/data/'
        :param bool overwrite: If False, and target files exist do not download.
            If True always download the remote files. Default False.
        :param bool skip_zero_size_check: If True no error will be raised for files with zero bytes size.
        :param bool silence_errors: If True, silence errors that might pop up when trying to downlaod
            files stored remotely. Default False

        :return: Path to downloaded file or None on error
        """
        if not local_folder:
            local_folder = CacheManager.get_cache_manager().get_cache_folder()
        local_path = os.path.join(
            str(Path(local_folder).absolute()), str(Path(urlparse(remote_url).path)).lstrip(os.path.sep)
        )
        helper = StorageHelper.get(remote_url)
        return helper.download_to_file(
            remote_url,
            local_path,
            overwrite_existing=overwrite,
            skip_zero_size_check=skip_zero_size_check,
            silence_errors=silence_errors,
        )

    @classmethod
    def exists_file(cls, remote_url):
        # type: (str) -> bool
        """
        Check if remote file exists. Note that this function will return
        False for directories.

        :param str remote_url: The url where the file is stored.
            E.g. 's3://bucket/some_file.txt', 'file://local/file'

        :return: True is the remote_url stores a file and False otherwise
        """
        if remote_url.startswith('file://'):
            return os.path.isfile(remote_url[len('file://'):])
        helper = StorageHelper.get(remote_url)
        obj = helper.get_object(remote_url)
        if not obj:
            return False
        return len(StorageManager.list(remote_url)) == 0

    @classmethod
    def get_file_size_bytes(cls, remote_url, silence_errors=False):
        # type: (str, bool) -> [int, None]
        """
        Get size of the remote file in bytes.

        :param str remote_url: The url where the file is stored.
            E.g. 's3://bucket/some_file.txt', 'file://local/file'
        :param bool silence_errors: Silence errors that might occur
            when fetching the size of the file. Default: False

        :return: The size of the file in bytes.
            None if the file could not be found or an error occurred.
        """
        helper = StorageHelper.get(remote_url)
        return helper.get_object_size_bytes(remote_url)

    @classmethod
    def download_folder(
        cls,
        remote_url,
        local_folder=None,
        match_wildcard=None,
        overwrite=False,
        skip_zero_size_check=False,
        silence_errors=False,
    ):
        # type: (str, Optional[str], Optional[str], bool, bool, bool) -> Optional[str]
        """
        Download remote folder recursively to the local machine, maintaining the sub folder structure
        from the remote storage.

        .. note::

            If we have a remote file `s3://bucket/sub/file.ext` then
            `StorageManager.download_folder('s3://bucket/', '~/folder/')`
            will create `~/folder/sub/file.ext`

        :param str remote_url: Source remote storage location, tree structure of `remote_url` will
            be created under the target local_folder. Supports S3/GS/Azure and shared filesystem.
            Example: 's3://bucket/data/'
        :param str local_folder: Local target folder to create the full tree from remote_url.
            If None, use the cache folder. (Default: use cache folder)
        :param match_wildcard: If specified only download files matching the `match_wildcard`
            Example: `*.json`
        :param bool overwrite: If False, and target files exist do not download.
            If True always download the remote files. Default False.
        :param bool skip_zero_size_check: If True no error will be raised for files with zero bytes size.
        :param bool silence_errors: If True, silence errors that might pop up when trying to downlaod
            files stored remotely. Default False

        :return: Target local folder
        """

        base_logger = LoggerRoot.get_base_logger()

        if local_folder:
            try:
                Path(local_folder).mkdir(parents=True, exist_ok=True)
            except OSError as ex:
                base_logger.error("Failed creating local folder '{}': {}".format(local_folder, ex))
                return
        else:
            local_folder = CacheManager.get_cache_manager().get_cache_folder()

        helper = StorageHelper.get(remote_url)
        results = []

        with ThreadPool() as pool:
            for path in helper.list(prefix=remote_url):
                remote_path = (
                    str(Path(helper.base_url) / Path(path))
                    if helper.get_driver_direct_access(helper.base_url)
                    else "{}/{}".format(helper.base_url.rstrip("/"), path.lstrip("/"))
                )
                if match_wildcard and not fnmatch.fnmatch(remote_path, match_wildcard):
                    continue
                results.append(
                    pool.apply_async(
                        cls.download_file,
                        args=(remote_path, local_folder),
                        kwds={
                            "overwrite": overwrite,
                            "skip_zero_size_check": skip_zero_size_check,
                            "silence_errors": silence_errors,
                        },
                    )
                )
            for res in results:
                res.wait()
        if not results and not silence_errors:
            LoggerRoot.get_base_logger().warning("Did not download any files matching {}".format(remote_url))
        return local_folder

    @classmethod
    def list(cls, remote_url, return_full_path=False):
        # type: (str, bool) -> Optional[List[str]]
        """
        Return a list of object names inside the base path

        :param str remote_url: The base path.
            For Google Storage, Azure and S3 it is the bucket of the path, for local files it is the root directory.
            For example: AWS S3: `s3://bucket/folder_` will list all the files you have in
            `s3://bucket-name/folder_*/*`. The same behaviour with Google Storage: `gs://bucket/folder_`,
            Azure blob storage: `azure://bucket/folder_` and also file system listing: `/mnt/share/folder_`
        :param bool return_full_path: If True, return a list of full object paths, otherwise return a list of
            relative object paths (default False).

        :return: The paths of all the objects in the storage base path under prefix, relative to the base path.
            None in case of list operation is not supported (http and https protocols for example)
        """
        helper = StorageHelper.get(remote_url)
        try:
            names_list = helper.list(prefix=remote_url)
        except Exception as ex:
            LoggerRoot.get_base_logger().warning("Can not list files for '{}' - {}".format(remote_url, ex))
            names_list = None

        if helper.base_url == 'file://':
            return ["{}/{}".format(remote_url.rstrip('/'), name) for name in names_list] if return_full_path else names_list
        return ["{}/{}".format(helper.base_url, name) for name in names_list] if return_full_path else names_list


# clearml/storage/util.py

import hashlib
import json
import re
import sys
from zlib import crc32
from typing import Optional, Union, Sequence, Dict
from pathlib2 import Path

from six.moves.urllib.parse import quote, urlparse, urlunparse
import six
import fnmatch

from ..debugging.log import LoggerRoot


def get_config_object_matcher(**patterns):
    unsupported = {k: v for k, v in patterns.items() if not isinstance(v, six.string_types)}
    if unsupported:
        raise ValueError('Unsupported object matcher (expecting string): %s'
                         % ', '.join('%s=%s' % (k, v) for k, v in unsupported.items()))

    # optimize simple patters
    starts_with = {k: v.rstrip('*') for k, v in patterns.items() if '*' not in v.rstrip('*') and '?' not in v}
    patterns = {k: v for k, v in patterns.items() if v not in starts_with}

    def _matcher(**kwargs):
        for key, value in kwargs.items():
            if not value:
                continue
            start = starts_with.get(key)
            if start:
                if value.startswith(start):
                    return True
            else:
                pat = patterns.get(key)
                if pat and fnmatch.fnmatch(value, pat):
                    return True

    return _matcher


def quote_url(url):
    parsed = urlparse(url)
    if parsed.scheme not in ("http", "https"):
        return url
    parsed = parsed._replace(path=quote(parsed.path))
    return urlunparse(parsed)


def encode_string_to_filename(text):
    return quote(text, safe=" ")


def sha256sum(filename, skip_header=0, block_size=65536):
    # type: (str, int, int) -> (Optional[str], Optional[str])
    # create sha2 of the file, notice we skip the header of the file (32 bytes)
    # because sometimes that is the only change
    h = hashlib.sha256()
    file_hash = hashlib.sha256()
    b = bytearray(block_size)
    mv = memoryview(b)
    try:
        with open(filename, 'rb', buffering=0) as f:
            # skip header
            if skip_header:
                file_hash.update(f.read(skip_header))
            # noinspection PyUnresolvedReferences
            for n in iter(lambda: f.readinto(mv), 0):
                h.update(mv[:n])
                if skip_header:
                    file_hash.update(mv[:n])
    except Exception as e:
        LoggerRoot.get_base_logger().warning(str(e))
        return None, None

    return h.hexdigest(), file_hash.hexdigest() if skip_header else None


def md5text(text, seed=1337):
    # type: (str, Union[int, str]) -> str
    """
    Return md5 hash of a string
    Do not use this hash for security, if needed use something stronger like SHA2

    :param text: string to hash
    :param seed: use prefix seed for hashing
    :return: md5 string
    """
    return hash_text(text=text, seed=seed, hash_func='md5')


def crc32text(text, seed=1337):
    # type: (str, Union[int, str]) -> str
    """
    Return crc32 hash of a string
    Do not use this hash for security, if needed use something stronger like SHA2

    :param text: string to hash
    :param seed: use prefix seed for hashing
    :return: crc32 hex in string (32bits = 8 characters in hex)
    """
    return '{:08x}'.format(crc32((str(seed)+str(text)).encode('utf-8')))


def hash_text(text, seed=1337, hash_func='md5'):
    # type: (str, Union[int, str], str) -> str
    """
    Return hash_func (md5/sha1/sha256/sha384/sha512) hash of a string

    :param text: string to hash
    :param seed: use prefix seed for hashing
    :param hash_func: hashing function. currently supported md5 sha256
    :return: hashed string
    """
    assert hash_func in ('md5', 'sha256', 'sha256', 'sha384', 'sha512')
    h = getattr(hashlib, hash_func)()
    h.update((str(seed) + str(text)).encode('utf-8'))
    return h.hexdigest()


def hash_dict(a_dict, seed=1337, hash_func='md5'):
    # type: (Dict, Union[int, str], str) -> str
    """
    Return hash_func (crc32/md5/sha1/sha256/sha384/sha512) hash of the dict values
    (dict must be JSON serializable)

    :param a_dict: a dictionary to hash
    :param seed: use prefix seed for hashing
    :param hash_func: hashing function. currently supported md5 sha256
    :return: hashed string
    """
    assert hash_func in ('crc32', 'md5', 'sha256', 'sha256', 'sha384', 'sha512')
    repr_string = json.dumps(a_dict, sort_keys=True)
    if hash_func == 'crc32':
        return crc32text(repr_string, seed=seed)
    else:
        return hash_text(repr_string, seed=seed, hash_func=hash_func)


def is_windows():
    """
    :return: True if currently running on windows OS
    """
    return sys.platform == 'win32'


def format_size(size_in_bytes, binary=False, use_nonbinary_notation=False, use_b_instead_of_bytes=False):
    # type: (Union[int, float], bool, bool, bool) -> str
    """
    Return the size in human readable format (string)
    Matching humanfriendly.format_size outputs

    :param size_in_bytes: number of bytes
    :param binary: If `True` 1 Kb equals 1024 bytes, if False (default) 1 KB = 1000 bytes
    :param use_nonbinary_notation: Only applies if binary is `True`. If this is `True`,
        the binary scale (KiB, MiB etc.) will be replaced with the regular scale (KB, MB etc.)
    :param use_b_instead_of_bytes: If `True`, return the formatted size with `B` as the
        scale instead of `byte(s)` (when applicable)
    :return: string representation of the number of bytes (b,Kb,Mb,Gb, Tb,)
        >>> format_size(0)
        '0 bytes'
        >>> format_size(1)
        '1 byte'
        >>> format_size(5)
        '5 bytes'
        > format_size(1000)
        '1 KB'
        > format_size(1024, binary=True)
        '1 KiB'
        >>> format_size(1000 ** 3 * 4)
        '4 GB'
    """
    size = float(size_in_bytes)
    # single byte is the exception here
    if size == 1 and not use_b_instead_of_bytes:
        return "{} byte".format(int(size))
    k = 1024 if binary else 1000
    scale = (
        ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB"]
        if (binary and not use_nonbinary_notation)
        else ["bytes", "KB", "MB", "GB", "TB", "PB"]
    )
    if use_b_instead_of_bytes:
        scale[0] = "B"
    for i, m in enumerate(scale):
        if size < k ** (i + 1) or i == len(scale) - 1:
            return (
                ("{:.2f}".format(size / (k ** i)).rstrip("0").rstrip(".") if i > 0 else "{}".format(int(size)))
                + " "
                + m
            )
    # we should never get here
    return "{} {}".format(int(size), scale[0])


def parse_size(size, binary=False):
    """
    Parse a human readable data size and return the number of bytes.
    Match humanfriendly.parse_size

    :param size: The human readable file size to parse (a string).
    :param binary: :data:`True` to use binary multiples of bytes (base-2) for
                   ambiguous unit symbols and names, :data:`False` to use
                   decimal multiples of bytes (base-10).
    :returns: The corresponding size in bytes (an integer).
    :raises: :exc:`InvalidSize` when the input can't be parsed.

    This function knows how to parse sizes in bytes, kilobytes, megabytes,
    gigabytes, terabytes and petabytes. Some examples:
        >>> parse_size('42')
        42
        >>> parse_size('13b')
        13
        >>> parse_size('5 bytes')
        5
        >>> parse_size('1 KB')
        1000
        >>> parse_size('1 kilobyte')
        1000
        >>> parse_size('1 KiB')
        1024
        >>> parse_size('1 KB', binary=True)
        1024
        >>> parse_size('1.5 GB')
        1500000000
        >>> parse_size('1.5 GB', binary=True)
        1610612736
    """
    def tokenize(text):
        tokenized_input = []
        for token in re.split(r'(\d+(?:\.\d+)?)', text):
            token = token.strip()
            if re.match(r'\d+\.\d+', token):
                tokenized_input.append(float(token))
            elif token.isdigit():
                tokenized_input.append(int(token))
            elif token:
                tokenized_input.append(token)
        return tokenized_input
    tokens = tokenize(str(size))
    if tokens and isinstance(tokens[0], (int, float)):
        disk_size_units_b = \
            (('B', 'bytes'), ('KiB', 'kibibyte'), ('MiB', 'mebibyte'), ('GiB', 'gibibyte'),
             ('TiB', 'tebibyte'), ('PiB', 'pebibyte'))
        disk_size_units_d = \
            (('B', 'bytes'), ('KB', 'kilobyte'), ('MB', 'megabyte'), ('GB', 'gigabyte'),
             ('TB', 'terabyte'), ('PB', 'petabyte'))
        disk_size_units_b = [(1024 ** i, s[0], s[1]) for i, s in enumerate(disk_size_units_b)]
        k = 1024 if binary else 1000
        disk_size_units_d = [(k ** i, s[0], s[1]) for i, s in enumerate(disk_size_units_d)]
        disk_size_units = (disk_size_units_b + disk_size_units_d) \
            if binary else (disk_size_units_d + disk_size_units_b)

        # Get the normalized unit (if any) from the tokenized input.
        normalized_unit = tokens[1].lower() if len(tokens) == 2 and isinstance(tokens[1], str) else ''
        # If the input contains only a number, it's assumed to be the number of
        # bytes. The second token can also explicitly reference the unit bytes.
        if len(tokens) == 1 or normalized_unit.startswith('b'):
            return int(tokens[0])
        # Otherwise we expect two tokens: A number and a unit.
        if normalized_unit:
            # Convert plural units to singular units, for details:
            # https://github.com/xolox/python-humanfriendly/issues/26
            normalized_unit = normalized_unit.rstrip('s')
            for k, low, high in disk_size_units:
                # First we check for unambiguous symbols (KiB, MiB, GiB, etc)
                # and names (kibibyte, mebibyte, gibibyte, etc) because their
                # handling is always the same.
                if normalized_unit in (low.lower(), high.lower()):
                    return int(tokens[0] * k)
                # Now we will deal with ambiguous prefixes (K, M, G, etc),
                # symbols (KB, MB, GB, etc) and names (kilobyte, megabyte,
                # gigabyte, etc) according to the caller's preference.
                if (normalized_unit in (low.lower(), high.lower()) or
                        normalized_unit.startswith(low.lower())):
                    return int(tokens[0] * k)

    raise ValueError("Failed to parse size! (input {} was tokenized as {})".format(size, tokens))


def get_common_path(list_of_files):
    # type: (Sequence[Union[str, Path]]) -> Optional[str]
    """
    Return the common path of a list of files

    :param list_of_files: list of files (str or Path objects)
    :return: Common path string (always absolute) or None if common path could not be found
    """
    if not list_of_files:
        return None

    # a single file has its parent as common path
    if len(list_of_files) == 1:
        return Path(list_of_files[0]).absolute().parent.as_posix()

    # find common path to support folder structure inside zip
    common_path_parts = Path(list_of_files[0]).absolute().parts
    for f in list_of_files:
        f_parts = Path(f).absolute().parts
        num_p = min(len(f_parts), len(common_path_parts))
        if f_parts[:num_p] == common_path_parts[:num_p]:
            common_path_parts = common_path_parts[:num_p]
            continue
        num_p = min(
            [i for i, (a, b) in enumerate(zip(common_path_parts[:num_p], f_parts[:num_p])) if a != b] or [-1])
        # no common path, break
        if num_p < 0:
            common_path_parts = []
            break
        # update common path
        common_path_parts = common_path_parts[:num_p]

    if common_path_parts:
        common_path = Path()
        for f in common_path_parts:
            common_path /= f
        return common_path.as_posix()

    return None


# setup.py

"""
ClearML - Artificial Intelligence Version Control
https://github.com/allegroai/clearml
"""

import os.path
# Always prefer setuptools over distutils
from setuptools import setup, find_packages


def read_text(filepath):
    with open(filepath, "r", encoding="utf-8") as f:
        return f.read()


here = os.path.dirname(__file__)
# Get the long description from the README file
long_description = read_text(os.path.join(here, 'README.md'))

# fix github, dark logo hack.
long_description = long_description.replace(
    """<img align="center" src="docs/clearml-logo.svg#gh-light-mode-only" alt="Clear|ML"><img align="center" src="docs/clearml-logo-dark.svg#gh-dark-mode-only" alt="Clear|ML">""",  # noqa
    """<a href="https://app.clear.ml"><img src="https://github.com/allegroai/clearml/blob/master/docs/clearml-logo.svg?raw=true" width="250px"></a>""",  # noqa
    1
)


def read_version_string(version_file):
    for line in read_text(version_file).splitlines():
        if line.startswith('__version__'):
            delim = '"' if '"' in line else "'"
            return line.split(delim)[1]
    else:
        raise RuntimeError("Unable to find version string.")


version = read_version_string("clearml/version.py")

requirements = read_text(os.path.join(here, 'requirements.txt')).splitlines()

setup(
    name='clearml',
    version=version,
    description='ClearML - Auto-Magical Experiment Manager, Version Control, and MLOps for AI',
    long_description=long_description,
    long_description_content_type='text/markdown',
    # The project's main homepage.
    url='https://github.com/allegroai/clearml',
    author='ClearML',
    author_email='support@clear.ml',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'Operating System :: POSIX :: Linux',
        'Operating System :: MacOS :: MacOS X',
        'Operating System :: Microsoft',
        'Topic :: Scientific/Engineering :: Artificial Intelligence',
        'Topic :: Software Development',
        'Topic :: Software Development :: Version Control',
        'Topic :: System :: Logging',
        'Topic :: System :: Monitoring',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
        'License :: OSI Approved :: Apache Software License',
    ],
    keywords='clearml trains development machine deep learning version control machine-learning machinelearning '
             'deeplearning deep-learning experiment-manager experimentmanager',
    packages=find_packages(exclude=['contrib', 'docs', 'data', 'examples', 'tests']),
    install_requires=requirements,
    extras_require={
        's3': [
            'boto3>=1.9',
        ],
        'azure': [
            'azure-storage-blob>=12.0.0',
        ],
        'gs': [
            'google-cloud-storage>=1.13.2',
        ],
    },
    package_data={
        'clearml': ['config/default/*.conf', 'backend_api/config/default/*.conf']
    },
    include_package_data=True,
    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={
        'console_scripts': [
            'clearml-init = clearml.cli.config.__main__:main',
            'clearml-data = clearml.cli.data.__main__:main',
            'clearml-task = clearml.cli.task.__main__:main',
            'clearml-param-search = clearml.cli.hpo.__main__:main'
        ],
    },
)


# clearml/utilities/gpu/gpustat.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Implementation of gpustat
@author Jongwook Choi
@url https://github.com/wookayin/gpustat

@ copied from gpu-stat 0.6.0
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import json
import platform
import sys
from datetime import datetime

import psutil
from ..gpu import pynvml as N

NOT_SUPPORTED = 'Not Supported'
MB = 1024 * 1024


class GPUStat(object):

    def __init__(self, entry):
        if not isinstance(entry, dict):
            raise TypeError(
                'entry should be a dict, {} given'.format(type(entry))
            )
        self.entry = entry

    def keys(self):
        return self.entry.keys()

    def __getitem__(self, key):
        return self.entry[key]

    @property
    def index(self):
        """
        Returns the index of GPU (as in nvidia-smi).
        """
        return self.entry['index']

    @property
    def uuid(self):
        """
        Returns the uuid returned by nvidia-smi,
        e.g. GPU-12345678-abcd-abcd-uuid-123456abcdef
        """
        return self.entry['uuid']

    @property
    def name(self):
        """
        Returns the name of GPU card (e.g. Geforce Titan X)
        """
        return self.entry['name']

    @property
    def memory_total(self):
        """
        Returns the total memory (in MB) as an integer.
        """
        return int(self.entry['memory.total'])

    @property
    def memory_used(self):
        """
        Returns the occupied memory (in MB) as an integer.
        """
        return int(self.entry['memory.used'])

    @property
    def memory_free(self):
        """
        Returns the free (available) memory (in MB) as an integer.
        """
        v = self.memory_total - self.memory_used
        return max(v, 0)

    @property
    def memory_available(self):
        """
        Returns the available memory (in MB) as an integer.
        Alias of memory_free.
        """
        return self.memory_free

    @property
    def temperature(self):
        """
        Returns the temperature (in celcius) of GPU as an integer,
        or None if the information is not available.
        """
        v = self.entry['temperature.gpu']
        return int(v) if v is not None else None

    @property
    def fan_speed(self):
        """
        Returns the fan speed percentage (0-100) of maximum intended speed
        as an integer, or None if the information is not available.
        """
        v = self.entry['fan.speed']
        return int(v) if v is not None else None

    @property
    def utilization(self):
        """
        Returns the GPU utilization (in percentile),
        or None if the information is not available.
        """
        v = self.entry['utilization.gpu']
        return int(v) if v is not None else None

    @property
    def power_draw(self):
        """
        Returns the GPU power usage in Watts,
        or None if the information is not available.
        """
        v = self.entry['power.draw']
        return int(v) if v is not None else None

    @property
    def power_limit(self):
        """
        Returns the (enforced) GPU power limit in Watts,
        or None if the information is not available.
        """
        v = self.entry['enforced.power.limit']
        return int(v) if v is not None else None

    @property
    def processes(self):
        """
        Get the list of running processes on the GPU.
        """
        return self.entry['processes']

    def jsonify(self):
        o = dict(self.entry)
        if self.entry['processes'] is not None:
            o['processes'] = [{k: v for (k, v) in p.items() if k != 'gpu_uuid'}
                              for p in self.entry['processes']]
        else:
            o['processes'] = '({})'.format(NOT_SUPPORTED)
        return o


class GPUStatCollection(object):
    global_processes = {}
    _initialized = False
    _device_count = None
    _gpu_device_info = {}

    def __init__(self, gpu_list, driver_version=None, driver_cuda_version=None):
        self.gpus = gpu_list

        # attach additional system information
        self.hostname = platform.node()
        self.query_time = datetime.now()
        self.driver_version = driver_version
        self.driver_cuda_version = driver_cuda_version

    @staticmethod
    def clean_processes():
        for pid in list(GPUStatCollection.global_processes.keys()):
            if not psutil.pid_exists(pid):
                del GPUStatCollection.global_processes[pid]

    @staticmethod
    def new_query(shutdown=False, per_process_stats=False, get_driver_info=False):
        """Query the information of all the GPUs on local machine"""
        initialized = False
        if not GPUStatCollection._initialized:
            N.nvmlInit()
            GPUStatCollection._initialized = True
            initialized = True

        def _decode(b):
            if isinstance(b, bytes):
                return b.decode()  # for python3, to unicode
            return b

        def get_gpu_info(index, handle):
            """Get one GPU information specified by nvml handle"""

            def get_process_info(nv_process):
                """Get the process information of specific pid"""
                process = {}
                if nv_process.pid not in GPUStatCollection.global_processes:
                    GPUStatCollection.global_processes[nv_process.pid] = \
                        psutil.Process(pid=nv_process.pid)
                process['pid'] = nv_process.pid
                # noinspection PyBroadException
                try:
                    # ps_process = GPUStatCollection.global_processes[nv_process.pid]
                    # we do not actually use these, so no point in collecting them
                    # process['username'] = ps_process.username()
                    # # cmdline returns full path;
                    # # as in `ps -o comm`, get short cmdnames.
                    # _cmdline = ps_process.cmdline()
                    # if not _cmdline:
                    #     # sometimes, zombie or unknown (e.g. [kworker/8:2H])
                    #     process['command'] = '?'
                    #     process['full_command'] = ['?']
                    # else:
                    #     process['command'] = os.path.basename(_cmdline[0])
                    #     process['full_command'] = _cmdline
                    # process['cpu_percent'] = ps_process.cpu_percent()
                    # process['cpu_memory_usage'] = \
                    #     round((ps_process.memory_percent() / 100.0) *
                    #           psutil.virtual_memory().total)
                    # Bytes to MBytes
                    process['gpu_memory_usage'] = nv_process.usedGpuMemory // MB
                except Exception:
                    # insufficient permissions
                    pass
                return process

            if not GPUStatCollection._gpu_device_info.get(index):
                name = _decode(N.nvmlDeviceGetName(handle))
                uuid = _decode(N.nvmlDeviceGetUUID(handle))
                GPUStatCollection._gpu_device_info[index] = (name, uuid)

            name, uuid = GPUStatCollection._gpu_device_info[index]

            try:
                temperature = N.nvmlDeviceGetTemperature(
                    handle, N.NVML_TEMPERATURE_GPU
                )
            except N.NVMLError:
                temperature = None  # Not supported

            try:
                fan_speed = N.nvmlDeviceGetFanSpeed(handle)
            except N.NVMLError:
                fan_speed = None  # Not supported

            try:
                memory = N.nvmlDeviceGetMemoryInfo(handle)  # in Bytes
            except N.NVMLError:
                memory = None  # Not supported

            try:
                utilization = N.nvmlDeviceGetUtilizationRates(handle)
            except N.NVMLError:
                utilization = None  # Not supported

            try:
                power = N.nvmlDeviceGetPowerUsage(handle)
            except N.NVMLError:
                power = None

            try:
                power_limit = N.nvmlDeviceGetEnforcedPowerLimit(handle)
            except N.NVMLError:
                power_limit = None

            try:
                nv_comp_processes = \
                    N.nvmlDeviceGetComputeRunningProcesses(handle)
            except N.NVMLError:
                nv_comp_processes = None  # Not supported
            try:
                nv_graphics_processes = \
                    N.nvmlDeviceGetGraphicsRunningProcesses(handle)
            except N.NVMLError:
                nv_graphics_processes = None  # Not supported

            if not per_process_stats or (nv_comp_processes is None and nv_graphics_processes is None):
                processes = None
            else:
                processes = []
                nv_comp_processes = nv_comp_processes or []
                nv_graphics_processes = nv_graphics_processes or []
                for nv_process in nv_comp_processes + nv_graphics_processes:
                    try:
                        process = get_process_info(nv_process)
                        processes.append(process)
                    except psutil.NoSuchProcess:
                        # TODO: add some reminder for NVML broken context
                        # e.g. nvidia-smi reset  or  reboot the system
                        pass

                # we do not actually use these, so no point in collecting them
                # # TODO: Do not block if full process info is not requested
                # time.sleep(0.1)
                # for process in processes:
                #     pid = process['pid']
                #     cache_process = GPUStatCollection.global_processes[pid]
                #     process['cpu_percent'] = cache_process.cpu_percent()

            index = N.nvmlDeviceGetIndex(handle)
            gpu_info = {
                'index': index,
                'uuid': uuid,
                'name': name,
                'temperature.gpu': temperature,
                'fan.speed': fan_speed,
                'utilization.gpu': utilization.gpu if utilization else None,
                'power.draw': power // 1000 if power is not None else None,
                'enforced.power.limit': power_limit // 1000
                if power_limit is not None else None,
                # Convert bytes into MBytes
                'memory.used': memory.used // MB if memory else None,
                'memory.total': memory.total // MB if memory else None,
                'processes': processes,
            }
            if per_process_stats:
                GPUStatCollection.clean_processes()
            return gpu_info

        # 1. get the list of gpu and status
        gpu_list = []
        if GPUStatCollection._device_count is None:
            GPUStatCollection._device_count = N.nvmlDeviceGetCount()

        for index in range(GPUStatCollection._device_count):
            handle = N.nvmlDeviceGetHandleByIndex(index)
            gpu_info = get_gpu_info(index, handle)
            gpu_stat = GPUStat(gpu_info)
            gpu_list.append(gpu_stat)

        # 2. additional info (driver version, etc).
        if get_driver_info:
            try:
                driver_version = _decode(N.nvmlSystemGetDriverVersion())
            except N.NVMLError:
                driver_version = None  # N/A

            # noinspection PyBroadException
            try:
                cuda_driver_version = str(N.nvmlSystemGetCudaDriverVersion())
            except BaseException:
                # noinspection PyBroadException
                try:
                    cuda_driver_version = str(N.nvmlSystemGetCudaDriverVersion_v2())
                except BaseException:
                    cuda_driver_version = None
            if cuda_driver_version:
                try:
                    cuda_driver_version = '{}.{}'.format(
                        int(cuda_driver_version)//1000, (int(cuda_driver_version) % 1000)//10)
                except (ValueError, TypeError):
                    pass
        else:
            driver_version = None
            cuda_driver_version = None

        # no need to shutdown:
        if shutdown and initialized:
            N.nvmlShutdown()
            GPUStatCollection._initialized = False

        return GPUStatCollection(gpu_list, driver_version=driver_version, driver_cuda_version=cuda_driver_version)

    def __len__(self):
        return len(self.gpus)

    def __iter__(self):
        return iter(self.gpus)

    def __getitem__(self, index):
        return self.gpus[index]

    def __repr__(self):
        s = 'GPUStatCollection(host=%s, [\n' % self.hostname
        s += '\n'.join('  ' + str(g) for g in self.gpus)
        s += '\n])'
        return s

    # --- Printing Functions ---
    def jsonify(self):
        return {
            'hostname': self.hostname,
            'query_time': self.query_time,
            "gpus": [g.jsonify() for g in self]
        }

    def print_json(self, fp=sys.stdout):
        def date_handler(obj):
            if hasattr(obj, 'isoformat'):
                return obj.isoformat()
            else:
                raise TypeError(type(obj))

        o = self.jsonify()
        json.dump(o, fp, indent=4, separators=(',', ': '),
                  default=date_handler)
        fp.write('\n')
        fp.flush()


def new_query(shutdown=False, per_process_stats=False, get_driver_info=False):
    '''
    Obtain a new GPUStatCollection instance by querying nvidia-smi
    to get the list of GPUs and running process information.
    '''
    return GPUStatCollection.new_query(shutdown=shutdown, per_process_stats=per_process_stats,
                                       get_driver_info=get_driver_info)


# clearml/utilities/gpu/pynvml.py

#####
# Copyright (c) 2011-2019, NVIDIA Corporation.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright notice,
#      this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright
#      notice, this list of conditions and the following disclaimer in the
#      documentation and/or other materials provided with the distribution.
#    * Neither the name of the NVIDIA Corporation nor the names of its
#      contributors may be used to endorse or promote products derived from
#      this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
# @ copied from nvidia-ml-py3 11.450.51
#####
# flake8: noqa
# This is only to ignore F405 errors

##
# Python bindings for the NVML library
##
from ctypes import *  # noqa: F403
import sys
import os
import threading
import string

# C Type mappings #
# Enums
_nvmlEnableState_t = c_uint
NVML_FEATURE_DISABLED = 0
NVML_FEATURE_ENABLED = 1

_nvmlBrandType_t = c_uint
NVML_BRAND_UNKNOWN = 0
NVML_BRAND_QUADRO = 1
NVML_BRAND_TESLA = 2
NVML_BRAND_NVS = 3
NVML_BRAND_GRID = 4
NVML_BRAND_GEFORCE = 5
NVML_BRAND_TITAN = 6
NVML_BRAND_COUNT = 7

_nvmlTemperatureThresholds_t = c_uint
NVML_TEMPERATURE_THRESHOLD_SHUTDOWN = 0
NVML_TEMPERATURE_THRESHOLD_SLOWDOWN = 1
NVML_TEMPERATURE_THRESHOLD_MEM_MAX = 2
NVML_TEMPERATURE_THRESHOLD_GPU_MAX = 3
NVML_TEMPERATURE_THRESHOLD_COUNT = 4

_nvmlTemperatureSensors_t = c_uint
NVML_TEMPERATURE_GPU = 0
NVML_TEMPERATURE_COUNT = 1

_nvmlComputeMode_t = c_uint
NVML_COMPUTEMODE_DEFAULT = 0
NVML_COMPUTEMODE_EXCLUSIVE_THREAD = 1  ## Support Removed
NVML_COMPUTEMODE_PROHIBITED = 2
NVML_COMPUTEMODE_EXCLUSIVE_PROCESS = 3
NVML_COMPUTEMODE_COUNT = 4

_nvmlMemoryLocation_t = c_uint
NVML_MEMORY_LOCATION_L1_CACHE = 0
NVML_MEMORY_LOCATION_L2_CACHE = 1
NVML_MEMORY_LOCATION_DEVICE_MEMORY = 2
NVML_MEMORY_LOCATION_DRAM = 2
NVML_MEMORY_LOCATION_REGISTER_FILE = 3
NVML_MEMORY_LOCATION_TEXTURE_MEMORY = 4
NVML_MEMORY_LOCATION_TEXTURE_SHM = 5
NVML_MEMORY_LOCATION_CBU = 6
NVML_MEMORY_LOCATION_SRAM = 7
NVML_MEMORY_LOCATION_COUNT = 8

NVML_NVLINK_MAX_LINKS = 12

# For backwards compatibility, maintain the incorrectly-named "LANES" define
NVML_NVLINK_MAX_LANES = NVML_NVLINK_MAX_LINKS

_nvmlNvLinkErrorCounter_t = c_uint
NVML_NVLINK_ERROR_DL_REPLAY = 0
NVML_NVLINK_ERROR_DL_RECOVERY = 1
NVML_NVLINK_ERROR_DL_CRC_FLIT = 2
NVML_NVLINK_ERROR_DL_CRC_DATA = 3
NVML_NVLINK_ERROR_COUNT = 4

_nvmlNvLinkCapability_t = c_uint
NVML_NVLINK_CAP_P2P_SUPPORTED = 0
NVML_NVLINK_CAP_SYSMEM_ACCESS = 1
NVML_NVLINK_CAP_P2P_ATOMICS = 2
NVML_NVLINK_CAP_SYSMEM_ATOMICS = 3
NVML_NVLINK_CAP_SLI_BRIDGE = 4
NVML_NVLINK_CAP_VALID = 5
NVML_NVLINK_CAP_COUNT = 6

_nvmlNvLinkUtilizationCountPktTypes_t = c_uint
NVML_NVLINK_COUNTER_PKTFILTER_NOP = 0x1
NVML_NVLINK_COUNTER_PKTFILTER_READ = 0x2
NVML_NVLINK_COUNTER_PKTFILTER_WRITE = 0x4
NVML_NVLINK_COUNTER_PKTFILTER_RATOM = 0x8
NVML_NVLINK_COUNTER_PKTFILTER_NRATOM = 0x10
NVML_NVLINK_COUNTER_PKTFILTER_FLUSH = 0x20
NVML_NVLINK_COUNTER_PKTFILTER_RESPDATA = 0x40
NVML_NVLINK_COUNTER_PKTFILTER_RESPNODATA = 0x80
NVML_NVLINK_COUNTER_PKTFILTER_ALL = 0xFF

_nvmlNvLinkUtilizationCountUnits_t = c_uint
NVML_NVLINK_COUNTER_UNIT_CYCLES = 0
NVML_NVLINK_COUNTER_UNIT_PACKETS = 1
NVML_NVLINK_COUNTER_UNIT_BYTES = 2
NVML_NVLINK_COUNTER_UNIT_RESERVED = 3
NVML_NVLINK_COUNTER_UNIT_COUNT = 4

# These are deprecated, instead use _nvmlMemoryErrorType_t
_nvmlEccBitType_t = c_uint
NVML_SINGLE_BIT_ECC = 0
NVML_DOUBLE_BIT_ECC = 1
NVML_ECC_ERROR_TYPE_COUNT = 2

_nvmlEccCounterType_t = c_uint
NVML_VOLATILE_ECC = 0
NVML_AGGREGATE_ECC = 1
NVML_ECC_COUNTER_TYPE_COUNT = 2

_nvmlMemoryErrorType_t = c_uint
NVML_MEMORY_ERROR_TYPE_CORRECTED = 0
NVML_MEMORY_ERROR_TYPE_UNCORRECTED = 1
NVML_MEMORY_ERROR_TYPE_COUNT = 2

_nvmlClockType_t = c_uint
NVML_CLOCK_GRAPHICS = 0
NVML_CLOCK_SM = 1
NVML_CLOCK_MEM = 2
NVML_CLOCK_VIDEO = 3
NVML_CLOCK_COUNT = 4

_nvmlClockId_t = c_uint
NVML_CLOCK_ID_CURRENT = 0
NVML_CLOCK_ID_APP_CLOCK_TARGET = 1
NVML_CLOCK_ID_APP_CLOCK_DEFAULT = 2
NVML_CLOCK_ID_CUSTOMER_BOOST_MAX = 3
NVML_CLOCK_ID_COUNT = 4

_nvmlDriverModel_t = c_uint
NVML_DRIVER_WDDM = 0
NVML_DRIVER_WDM = 1

_nvmlPstates_t = c_uint
NVML_PSTATE_0 = 0
NVML_PSTATE_1 = 1
NVML_PSTATE_2 = 2
NVML_PSTATE_3 = 3
NVML_PSTATE_4 = 4
NVML_PSTATE_5 = 5
NVML_PSTATE_6 = 6
NVML_PSTATE_7 = 7
NVML_PSTATE_8 = 8
NVML_PSTATE_9 = 9
NVML_PSTATE_10 = 10
NVML_PSTATE_11 = 11
NVML_PSTATE_12 = 12
NVML_PSTATE_13 = 13
NVML_PSTATE_14 = 14
NVML_PSTATE_15 = 15
NVML_PSTATE_UNKNOWN = 32

_nvmlInforomObject_t = c_uint
NVML_INFOROM_OEM = 0
NVML_INFOROM_ECC = 1
NVML_INFOROM_POWER = 2
NVML_INFOROM_COUNT = 3

_nvmlReturn_t = c_uint
NVML_SUCCESS = 0
NVML_ERROR_UNINITIALIZED = 1
NVML_ERROR_INVALID_ARGUMENT = 2
NVML_ERROR_NOT_SUPPORTED = 3
NVML_ERROR_NO_PERMISSION = 4
NVML_ERROR_ALREADY_INITIALIZED = 5
NVML_ERROR_NOT_FOUND = 6
NVML_ERROR_INSUFFICIENT_SIZE = 7
NVML_ERROR_INSUFFICIENT_POWER = 8
NVML_ERROR_DRIVER_NOT_LOADED = 9
NVML_ERROR_TIMEOUT = 10
NVML_ERROR_IRQ_ISSUE = 11
NVML_ERROR_LIBRARY_NOT_FOUND = 12
NVML_ERROR_FUNCTION_NOT_FOUND = 13
NVML_ERROR_CORRUPTED_INFOROM = 14
NVML_ERROR_GPU_IS_LOST = 15
NVML_ERROR_RESET_REQUIRED = 16
NVML_ERROR_OPERATING_SYSTEM = 17
NVML_ERROR_LIB_RM_VERSION_MISMATCH = 18
NVML_ERROR_IN_USE = 19
NVML_ERROR_MEMORY = 20
NVML_ERROR_NO_DATA = 21
NVML_ERROR_VGPU_ECC_NOT_SUPPORTED = 22
NVML_ERROR_INSUFFICIENT_RESOURCES = 23
NVML_ERROR_UNKNOWN = 999

_nvmlFanState_t = c_uint
NVML_FAN_NORMAL = 0
NVML_FAN_FAILED = 1

_nvmlLedColor_t = c_uint
NVML_LED_COLOR_GREEN = 0
NVML_LED_COLOR_AMBER = 1

_nvmlGpuOperationMode_t = c_uint
NVML_GOM_ALL_ON = 0
NVML_GOM_COMPUTE = 1
NVML_GOM_LOW_DP = 2

_nvmlPageRetirementCause_t = c_uint
NVML_PAGE_RETIREMENT_CAUSE_MULTIPLE_SINGLE_BIT_ECC_ERRORS = 0
NVML_PAGE_RETIREMENT_CAUSE_DOUBLE_BIT_ECC_ERROR = 1
NVML_PAGE_RETIREMENT_CAUSE_COUNT = 2

_nvmlRestrictedAPI_t = c_uint
NVML_RESTRICTED_API_SET_APPLICATION_CLOCKS = 0
NVML_RESTRICTED_API_SET_AUTO_BOOSTED_CLOCKS = 1
NVML_RESTRICTED_API_COUNT = 2

_nvmlBridgeChipType_t = c_uint
NVML_BRIDGE_CHIP_PLX = 0
NVML_BRIDGE_CHIP_BRO4 = 1
NVML_MAX_PHYSICAL_BRIDGE = 128

_nvmlValueType_t = c_uint
NVML_VALUE_TYPE_DOUBLE = 0
NVML_VALUE_TYPE_UNSIGNED_INT = 1
NVML_VALUE_TYPE_UNSIGNED_LONG = 2
NVML_VALUE_TYPE_UNSIGNED_LONG_LONG = 3
NVML_VALUE_TYPE_SIGNED_LONG_LONG = 4
NVML_VALUE_TYPE_COUNT = 5

_nvmlPerfPolicyType_t = c_uint
NVML_PERF_POLICY_POWER = 0
NVML_PERF_POLICY_THERMAL = 1
NVML_PERF_POLICY_SYNC_BOOST = 2
NVML_PERF_POLICY_BOARD_LIMIT = 3
NVML_PERF_POLICY_LOW_UTILIZATION = 4
NVML_PERF_POLICY_RELIABILITY = 5
NVML_PERF_POLICY_TOTAL_APP_CLOCKS = 10
NVML_PERF_POLICY_TOTAL_BASE_CLOCKS = 11
NVML_PERF_POLICY_COUNT = 12

_nvmlEncoderQueryType_t = c_uint
NVML_ENCODER_QUERY_H264 = 0
NVML_ENCODER_QUERY_HEVC = 1

_nvmlFBCSessionType_t = c_uint
NVML_FBC_SESSION_TYPE_UNKNOWN = 0
NVML_FBC_SESSION_TYPE_TOSYS = 1
NVML_FBC_SESSION_TYPE_CUDA = 2
NVML_FBC_SESSION_TYPE_VID = 3
NVML_FBC_SESSION_TYPE_HWENC = 4

_nvmlDetachGpuState_t = c_uint
NVML_DETACH_GPU_KEEP = 0
NVML_DETACH_GPU_REMOVE = 1

_nvmlPcieLinkState_t = c_uint
NVML_PCIE_LINK_KEEP = 0
NVML_PCIE_LINK_SHUT_DOWN = 1

_nvmlSamplingType_t = c_uint
NVML_TOTAL_POWER_SAMPLES = 0
NVML_GPU_UTILIZATION_SAMPLES = 1
NVML_MEMORY_UTILIZATION_SAMPLES = 2
NVML_ENC_UTILIZATION_SAMPLES = 3
NVML_DEC_UTILIZATION_SAMPLES = 4
NVML_PROCESSOR_CLK_SAMPLES = 5
NVML_MEMORY_CLK_SAMPLES = 6
NVML_SAMPLINGTYPE_COUNT = 7

_nvmlPcieUtilCounter_t = c_uint
NVML_PCIE_UTIL_TX_BYTES = 0
NVML_PCIE_UTIL_RX_BYTES = 1
NVML_PCIE_UTIL_COUNT = 2

_nvmlGpuTopologyLevel_t = c_uint
NVML_TOPOLOGY_INTERNAL = 0
NVML_TOPOLOGY_SINGLE = 10
NVML_TOPOLOGY_MULTIPLE = 20
NVML_TOPOLOGY_HOSTBRIDGE = 30
NVML_TOPOLOGY_NODE = 40
NVML_TOPOLOGY_CPU = NVML_TOPOLOGY_NODE
NVML_TOPOLOGY_SYSTEM = 50

_nvmlGpuP2PCapsIndex_t = c_uint
NVML_P2P_CAPS_INDEX_READ = 0,
NVML_P2P_CAPS_INDEX_WRITE = 1
NVML_P2P_CAPS_INDEX_NVLINK = 2
NVML_P2P_CAPS_INDEX_ATOMICS = 3
NVML_P2P_CAPS_INDEX_PROP = 4
NVML_P2P_CAPS_INDEX_LOOPBACK = 5
NVML_P2P_CAPS_INDEX_UNKNOWN = 6

_nvmlGpuP2PStatus_t = c_uint
NVML_P2P_STATUS_OK = 0
NVML_P2P_STATUS_CHIPSET_NOT_SUPPORED = 1
NVML_P2P_STATUS_GPU_NOT_SUPPORTED = 2
NVML_P2P_STATUS_IOH_TOPOLOGY_NOT_SUPPORTED = 3
NVML_P2P_STATUS_DISABLED_BY_REGKEY = 4
NVML_P2P_STATUS_NOT_SUPPORTED = 5
NVML_P2P_STATUS_UNKNOWN = 6

_nvmlDeviceArchitecture_t = c_uint
NVML_DEVICE_ARCH_KEPLER = 2
NVML_DEVICE_ARCH_MAXWELL = 3
NVML_DEVICE_ARCH_PASCAL = 4
NVML_DEVICE_ARCH_VOLTA = 5
NVML_DEVICE_ARCH_TURING = 6
NVML_DEVICE_ARCH_AMPERE = 7
NVML_DEVICE_ARCH_UNKNOWN = 0xffffffff

_nvmlClockLimitId_t = c_uint
NVML_CLOCK_LIMIT_ID_RANGE_START = 0xffffff00
NVML_CLOCK_LIMIT_ID_TDP = 0xffffff01
NVML_CLOCK_LIMIT_ID_UNLIMITED = 0xffffff02

_nvmlAffinityScope_t = c_uint
NVML_AFFINITY_SCOPE_NODE = 0
NVML_AFFINITY_SCOPE_SOCKET = 1

# C preprocessor defined values
nvmlFlagDefault = 0
nvmlFlagForce = 1
NVML_INIT_FLAG_NO_GPUS = 1
NVML_INIT_FLAG_NO_ATTACH = 2

NVML_MAX_GPC_COUNT = 32

# buffer size
NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE = 16
NVML_DEVICE_UUID_BUFFER_SIZE = 80
NVML_DEVICE_UUID_V2_BUFFER_SIZE = 96
NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE = 80
NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE = 80
NVML_DEVICE_NAME_BUFFER_SIZE = 64
NVML_DEVICE_SERIAL_BUFFER_SIZE = 30
NVML_DEVICE_PART_NUMBER_BUFFER_SIZE = 80
NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE = 32
NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE = 32
NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE = 16
NVML_GRID_LICENSE_BUFFER_SIZE = 128
NVML_VGPU_NAME_BUFFER_SIZE = 64
NVML_GRID_LICENSE_FEATURE_MAX_COUNT = 3
NVML_VGPU_METADATA_OPAQUE_DATA_SIZE = sizeof(c_uint) + 256
NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE = 256

# Format strings
NVML_DEVICE_PCI_BUS_ID_LEGACY_FMT = "%04X:%02X:%02X.0"
NVML_DEVICE_PCI_BUS_ID_FMT = "%08X:%02X:%02X.0"

NVML_VALUE_NOT_AVAILABLE_ulonglong = c_ulonglong(-1)
NVML_VALUE_NOT_AVAILABLE_uint = c_uint(-1)

'''
 Field Identifiers.

 All Identifiers pertain to a device. Each ID is only used once and is guaranteed never to change.
'''
NVML_FI_DEV_ECC_CURRENT = 1  # Current ECC mode. 1=Active. 0=Inactive
NVML_FI_DEV_ECC_PENDING = 2  # Pending ECC mode. 1=Active. 0=Inactive

# ECC Count Totals
NVML_FI_DEV_ECC_SBE_VOL_TOTAL = 3  # Total single bit volatile ECC errors
NVML_FI_DEV_ECC_DBE_VOL_TOTAL = 4  # Total double bit volatile ECC errors
NVML_FI_DEV_ECC_SBE_AGG_TOTAL = 5  # Total single bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_DBE_AGG_TOTAL = 6  # Total double bit aggregate (persistent) ECC errors
# Individual ECC locations
NVML_FI_DEV_ECC_SBE_VOL_L1 = 7  # L1 cache single bit volatile ECC errors
NVML_FI_DEV_ECC_DBE_VOL_L1 = 8  # L1 cache double bit volatile ECC errors
NVML_FI_DEV_ECC_SBE_VOL_L2 = 9  # L2 cache single bit volatile ECC errors
NVML_FI_DEV_ECC_DBE_VOL_L2 = 10  # L2 cache double bit volatile ECC errors
NVML_FI_DEV_ECC_SBE_VOL_DEV = 11  # Device memory single bit volatile ECC errors
NVML_FI_DEV_ECC_DBE_VOL_DEV = 12  # Device memory double bit volatile ECC errors
NVML_FI_DEV_ECC_SBE_VOL_REG = 13  # Register file single bit volatile ECC errors
NVML_FI_DEV_ECC_DBE_VOL_REG = 14  # Register file double bit volatile ECC errors
NVML_FI_DEV_ECC_SBE_VOL_TEX = 15  # Texture memory single bit volatile ECC errors
NVML_FI_DEV_ECC_DBE_VOL_TEX = 16  # Texture memory double bit volatile ECC errors
NVML_FI_DEV_ECC_DBE_VOL_CBU = 17  # CBU double bit volatile ECC errors
NVML_FI_DEV_ECC_SBE_AGG_L1 = 18  # L1 cache single bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_DBE_AGG_L1 = 19  # L1 cache double bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_SBE_AGG_L2 = 20  # L2 cache single bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_DBE_AGG_L2 = 21  # L2 cache double bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_SBE_AGG_DEV = 22  # Device memory single bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_DBE_AGG_DEV = 23  # Device memory double bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_SBE_AGG_REG = 24  # Register File single bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_DBE_AGG_REG = 25  # Register File double bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_SBE_AGG_TEX = 26  # Texture memory single bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_DBE_AGG_TEX = 27  # Texture memory double bit aggregate (persistent) ECC errors
NVML_FI_DEV_ECC_DBE_AGG_CBU = 28  # CBU double bit aggregate ECC errors

# Page Retirement
NVML_FI_DEV_RETIRED_SBE = 29  # Number of retired pages because of single bit errors
NVML_FI_DEV_RETIRED_DBE = 30  # Number of retired pages because of double bit errors
NVML_FI_DEV_RETIRED_PENDING = 31  # If any pages are pending retirement. 1=yes. 0=no.

# NvLink Flit Error Counters
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L0 = 32  # NVLink flow control CRC  Error Counter for Lane 0
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L1 = 33  # NVLink flow control CRC  Error Counter for Lane 1
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L2 = 34  # NVLink flow control CRC  Error Counter for Lane 2
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L3 = 35  # NVLink flow control CRC  Error Counter for Lane 3
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L4 = 36  # NVLink flow control CRC  Error Counter for Lane 4
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L5 = 37  # NVLink flow control CRC  Error Counter for Lane 5
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL = 38  # NVLink flow control CRC  Error Counter total for all Lanes

# NvLink CRC Data Error Counters
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L0 = 39  # NVLink data CRC Error Counter for Lane 0
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L1 = 40  # NVLink data CRC Error Counter for Lane 1
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L2 = 41  # NVLink data CRC Error Counter for Lane 2
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L3 = 42  # NVLink data CRC Error Counter for Lane 3
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L4 = 43  # NVLink data CRC Error Counter for Lane 4
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L5 = 44  # NVLink data CRC Error Counter for Lane 5
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL = 45  # NvLink data CRC Error Counter total for all Lanes

# NvLink Replay Error Counters
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L0 = 46  # NVLink Replay Error Counter for Lane 0
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L1 = 47  # NVLink Replay Error Counter for Lane 1
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L2 = 48  # NVLink Replay Error Counter for Lane 2
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L3 = 49  # NVLink Replay Error Counter for Lane 3
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L4 = 50  # NVLink Replay Error Counter for Lane 4
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L5 = 51  # NVLink Replay Error Counter for Lane 5
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL = 52  # NVLink Replay Error Counter total for all Lanes

# NvLink Recovery Error Counters
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L0 = 53  # NVLink Recovery Error Counter for Lane 0
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L1 = 54  # NVLink Recovery Error Counter for Lane 1
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L2 = 55  # NVLink Recovery Error Counter for Lane 2
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L3 = 56  # NVLink Recovery Error Counter for Lane 3
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L4 = 57  # NVLink Recovery Error Counter for Lane 4
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L5 = 58  # NVLink Recovery Error Counter for Lane 5
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL = 59  # NVLink Recovery Error Counter total for all Lanes

# NvLink Bandwidth Counters
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L0 = 60  # NVLink Bandwidth Counter for Counter Set 0, Lane 0
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L1 = 61  # NVLink Bandwidth Counter for Counter Set 0, Lane 1
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L2 = 62  # NVLink Bandwidth Counter for Counter Set 0, Lane 2
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L3 = 63  # NVLink Bandwidth Counter for Counter Set 0, Lane 3
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L4 = 64  # NVLink Bandwidth Counter for Counter Set 0, Lane 4
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L5 = 65  # NVLink Bandwidth Counter for Counter Set 0, Lane 5
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_TOTAL = 66  # NVLink Bandwidth Counter Total for Counter Set 0, All Lanes

# NvLink Bandwidth Counters
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L0 = 67  # NVLink Bandwidth Counter for Counter Set 1, Lane 0
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L1 = 68  # NVLink Bandwidth Counter for Counter Set 1, Lane 1
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L2 = 69  # NVLink Bandwidth Counter for Counter Set 1, Lane 2
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L3 = 70  # NVLink Bandwidth Counter for Counter Set 1, Lane 3
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L4 = 71  # NVLink Bandwidth Counter for Counter Set 1, Lane 4
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L5 = 72  # NVLink Bandwidth Counter for Counter Set 1, Lane 5
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_TOTAL = 73  # NVLink Bandwidth Counter Total for Counter Set 1, All Lanes

# Perf Policy Counters
NVML_FI_DEV_PERF_POLICY_POWER = 74  # Perf Policy Counter for Power Policy
NVML_FI_DEV_PERF_POLICY_THERMAL = 75  # Perf Policy Counter for Thermal Policy
NVML_FI_DEV_PERF_POLICY_SYNC_BOOST = 76  # Perf Policy Counter for Sync boost Policy
NVML_FI_DEV_PERF_POLICY_BOARD_LIMIT = 77  # Perf Policy Counter for Board Limit
NVML_FI_DEV_PERF_POLICY_LOW_UTILIZATION = 78  # Perf Policy Counter for Low GPU Utilization Policy
NVML_FI_DEV_PERF_POLICY_RELIABILITY = 79  # Perf Policy Counter for Reliability Policy
NVML_FI_DEV_PERF_POLICY_TOTAL_APP_CLOCKS = 80  # Perf Policy Counter for Total App Clock Policy
NVML_FI_DEV_PERF_POLICY_TOTAL_BASE_CLOCKS = 81  # Perf Policy Counter for Total Base Clocks Policy

# Memory temperatures
NVML_FI_DEV_MEMORY_TEMP = 82  # Memory temperature for the device

# Energy Counter
NVML_FI_DEV_TOTAL_ENERGY_CONSUMPTION = 83  # Total energy consumption for the GPU in mJ since the driver was last
# reloaded

# NVLink Speed
NVML_FI_DEV_NVLINK_SPEED_MBPS_L0 = 84
NVML_FI_DEV_NVLINK_SPEED_MBPS_L1 = 85
NVML_FI_DEV_NVLINK_SPEED_MBPS_L2 = 86
NVML_FI_DEV_NVLINK_SPEED_MBPS_L3 = 87
NVML_FI_DEV_NVLINK_SPEED_MBPS_L4 = 88
NVML_FI_DEV_NVLINK_SPEED_MBPS_L5 = 89
NVML_FI_DEV_NVLINK_SPEED_MBPS_COMMON = 90

# NVLink Link Count
NVML_FI_DEV_NVLINK_LINK_COUNT = 91

# Page Retirement pending fields
NVML_FI_DEV_RETIRED_PENDING_SBE = 92
NVML_FI_DEV_RETIRED_PENDING_DBE = 93

# PCIe replay and replay rollover counters
NVML_FI_DEV_PCIE_REPLAY_COUNTER = 94
NVML_FI_DEV_PCIE_REPLAY_ROLLOVER_COUNTER = 95

# NvLink Flit Error Counters
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L6 = 96  # NVLink flow control CRC  Error Counter for Lane 6
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L7 = 97  # NVLink flow control CRC  Error Counter for Lane 7
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L8 = 98  # NVLink flow control CRC  Error Counter for Lane 8
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L9 = 99  # NVLink flow control CRC  Error Counter for Lane 9
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L10 = 100  # NVLink flow control CRC  Error Counter for Lane 10
NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L11 = 101  # NVLink flow control CRC  Error Counter for Lane 11

# NvLink CRC Data Error Counters
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L6 = 102  # NVLink data CRC Error Counter for Lane 6
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L7 = 103  # NVLink data CRC Error Counter for Lane 7
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L8 = 104  # NVLink data CRC Error Counter for Lane 8
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L9 = 105  # NVLink data CRC Error Counter for Lane 9
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L10 = 106  # NVLink data CRC Error Counter for Lane 10
NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L11 = 107  # NVLink data CRC Error Counter for Lane 11

# NvLink Replay Error Counters
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L6 = 108  # NVLink Replay Error Counter for Lane 6
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L7 = 109  # NVLink Replay Error Counter for Lane 7
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L8 = 110  # NVLink Replay Error Counter for Lane 8
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L9 = 111  # NVLink Replay Error Counter for Lane 9
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L10 = 112  # NVLink Replay Error Counter for Lane 10
NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L11 = 113  # NVLink Replay Error Counter for Lane 11

# NvLink Recovery Error Counters
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L6 = 114  # NVLink Recovery Error Counter for Lane 6
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L7 = 115  # NVLink Recovery Error Counter for Lane 7
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L8 = 116  # NVLink Recovery Error Counter for Lane 8
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L9 = 117  # NVLink Recovery Error Counter for Lane 9
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L10 = 118  # NVLink Recovery Error Counter for Lane 10
NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L11 = 119  # NVLink Recovery Error Counter for Lane 11

# NvLink Bandwidth Counters
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L6 = 120  # NVLink Bandwidth Counter for Counter Set 0, Lane 6
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L7 = 121  # NVLink Bandwidth Counter for Counter Set 0, Lane 7
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L8 = 122  # NVLink Bandwidth Counter for Counter Set 0, Lane 8
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L9 = 123  # NVLink Bandwidth Counter for Counter Set 0, Lane 9
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L10 = 124  # NVLink Bandwidth Counter for Counter Set 0, Lane 10
NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L11 = 125  # NVLink Bandwidth Counter for Counter Set 0, Lane 11

# NvLink Bandwidth Counters
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L6 = 126  # NVLink Bandwidth Counter for Counter Set 1, Lane 6
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L7 = 127  # NVLink Bandwidth Counter for Counter Set 1, Lane 7
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L8 = 128  # NVLink Bandwidth Counter for Counter Set 1, Lane 8
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L9 = 129  # NVLink Bandwidth Counter for Counter Set 1, Lane 9
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L10 = 130  # NVLink Bandwidth Counter for Counter Set 1, Lane 10
NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L11 = 131  # NVLink Bandwidth Counter for Counter Set 1, Lane 11

# NVLink Speed
NVML_FI_DEV_NVLINK_SPEED_MBPS_L6 = 132
NVML_FI_DEV_NVLINK_SPEED_MBPS_L7 = 133
NVML_FI_DEV_NVLINK_SPEED_MBPS_L8 = 134
NVML_FI_DEV_NVLINK_SPEED_MBPS_L9 = 135
NVML_FI_DEV_NVLINK_SPEED_MBPS_L10 = 136
NVML_FI_DEV_NVLINK_SPEED_MBPS_L11 = 137

# NVLink Throughput Counters
NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_TX = 138  # NVLink TX Data throughput in KiB
NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_RX = 139  # NVLink RX Data throughput in KiB
NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_TX = 140  # NVLink TX Data + protocol overhead in KiB
NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_RX = 141  # NVLink RX Data + protocol overhead in KiB

# Row Remapper
NVML_FI_DEV_REMAPPED_COR = 142
NVML_FI_DEV_REMAPPED_UNC = 143
NVML_FI_DEV_REMAPPED_PENDING = 144
NVML_FI_DEV_REMAPPED_FAILURE = 145

NVML_FI_MAX = 146  # One greater than the largest field ID defined above

## Enums needed for the method nvmlDeviceGetVirtualizationMode and nvmlDeviceSetVirtualizationMode
NVML_GPU_VIRTUALIZATION_MODE_NONE = 0  # Represents Bare Metal GPU
NVML_GPU_VIRTUALIZATION_MODE_PASSTHROUGH = 1  # Device is associated with GPU-Passthorugh
NVML_GPU_VIRTUALIZATION_MODE_VGPU = 2  # Device is associated with vGPU inside virtual machine.
NVML_GPU_VIRTUALIZATION_MODE_HOST_VGPU = 3  # Device is associated with VGX hypervisor in vGPU mode
NVML_GPU_VIRTUALIZATION_MODE_HOST_VSGA = 4  # Device is associated with VGX hypervisor in vSGA mode

## Lib loading ##
nvmlLib = None
libLoadLock = threading.Lock()
_nvmlLib_refcount = 0  # Incremented on each nvmlInit and decremented on nvmlShutdown

## vGPU Management
_nvmlVgpuTypeId_t = c_uint
_nvmlVgpuInstance_t = c_uint

_nvmlVgpuVmIdType_t = c_uint
NVML_VGPU_VM_ID_DOMAIN_ID = 0
NVML_VGPU_VM_ID_UUID = 1

_nvmlGridLicenseFeatureCode_t = c_uint
NVML_GRID_LICENSE_FEATURE_CODE_VGPU = 1
NVML_GRID_LICENSE_FEATURE_CODE_VWORKSTATION = 2

_nvmlVgpuGuestInfoState_t = c_uint
NVML_VGPU_INSTANCE_GUEST_INFO_STATE_UNINITIALIZED = 0
NVML_VGPU_INSTANCE_GUEST_INFO_STATE_INITIALIZED = 1

_nvmlVgpuVmCompatibility_t = c_uint
NVML_VGPU_VM_COMPATIBILITY_NONE = 0x0
NVML_VGPU_VM_COMPATIBILITY_COLD = 0x1
NVML_VGPU_VM_COMPATIBILITY_HIBERNATE = 0x2
NVML_VGPU_VM_COMPATIBILITY_SLEEP = 0x4
NVML_VGPU_VM_COMPATIBILITY_LIVE = 0x8

_nvmlVgpuPgpuCompatibilityLimitCode_t = c_uint
NVML_VGPU_COMPATIBILITY_LIMIT_NONE = 0x0
NVML_VGPU_COMPATIBILITY_LIMIT_HOST_DRIVER = 0x1
NVML_VGPU_COMPATIBILITY_LIMIT_GUEST_DRIVER = 0x2
NVML_VGPU_COMPATIBILITY_LIMIT_GPU = 0x4
NVML_VGPU_COMPATIBILITY_LIMIT_OTHER = 0x80000000

_nvmlHostVgpuMode_t = c_uint
NVML_HOST_VGPU_MODE_NON_SRIOV = 0
NVML_HOST_VGPU_MODE_SRIOV = 1


## Error Checking ##
class NVMLError(Exception):
    _valClassMapping = dict()
    # List of currently known error codes
    _errcode_to_string = {
        NVML_ERROR_UNINITIALIZED: "Uninitialized",
        NVML_ERROR_INVALID_ARGUMENT: "Invalid Argument",
        NVML_ERROR_NOT_SUPPORTED: "Not Supported",
        NVML_ERROR_NO_PERMISSION: "Insufficient Permissions",
        NVML_ERROR_ALREADY_INITIALIZED: "Already Initialized",
        NVML_ERROR_NOT_FOUND: "Not Found",
        NVML_ERROR_INSUFFICIENT_SIZE: "Insufficient Size",
        NVML_ERROR_INSUFFICIENT_POWER: "Insufficient External Power",
        NVML_ERROR_DRIVER_NOT_LOADED: "Driver Not Loaded",
        NVML_ERROR_TIMEOUT: "Timeout",
        NVML_ERROR_IRQ_ISSUE: "Interrupt Request Issue",
        NVML_ERROR_LIBRARY_NOT_FOUND: "NVML Shared Library Not Found",
        NVML_ERROR_FUNCTION_NOT_FOUND: "Function Not Found",
        NVML_ERROR_CORRUPTED_INFOROM: "Corrupted infoROM",
        NVML_ERROR_GPU_IS_LOST: "GPU is lost",
        NVML_ERROR_RESET_REQUIRED: "GPU requires restart",
        NVML_ERROR_OPERATING_SYSTEM: "The operating system has blocked the request.",
        NVML_ERROR_LIB_RM_VERSION_MISMATCH: "RM has detected an NVML/RM version mismatch.",
        NVML_ERROR_MEMORY: "Insufficient Memory",
        NVML_ERROR_UNKNOWN: "Unknown Error",
    }

    def __new__(typ, value):
        '''
        Maps value to a proper subclass of NVMLError.
        See _extractNVMLErrorsAsClasses function for more details
        '''
        if typ == NVMLError:
            typ = NVMLError._valClassMapping.get(value, typ)
        obj = Exception.__new__(typ)
        obj.value = value
        return obj

    def __str__(self):
        try:
            if self.value not in NVMLError._errcode_to_string:
                NVMLError._errcode_to_string[self.value] = str(nvmlErrorString(self.value))
            return NVMLError._errcode_to_string[self.value]
        except NVMLError:   # NVMLError_Uninitialized:
            return "NVML Error with code %d" % self.value

    def __eq__(self, other):
        return self.value == other.value


def nvmlExceptionClass(nvmlErrorCode):
    if nvmlErrorCode not in NVMLError._valClassMapping:
        raise ValueError('nvmlErrorCode %s is not valid' % nvmlErrorCode)
    return NVMLError._valClassMapping[nvmlErrorCode]


def _extractNVMLErrorsAsClasses():
    '''
    Generates a hierarchy of classes on top of NVMLError class.

    Each NVML Error gets a new NVMLError subclass. This way try,except blocks can filter appropriate
    exceptions more easily.

    NVMLError is a parent class. Each NVML_ERROR_* gets it's own subclass.
    e.g. NVML_ERROR_ALREADY_INITIALIZED will be turned into NVMLError_AlreadyInitialized
    '''
    this_module = sys.modules[__name__]
    nvmlErrorsNames = [x for x in dir(this_module) if x.startswith("NVML_ERROR_")]
    for err_name in nvmlErrorsNames:
        # e.g. Turn NVML_ERROR_ALREADY_INITIALIZED into NVMLError_AlreadyInitialized
        class_name = "NVMLError_" + string.capwords(err_name.replace("NVML_ERROR_", ""), "_").replace("_", "")
        err_val = getattr(this_module, err_name)

        def gen_new(val):
            def new(typ):
                obj = NVMLError.__new__(typ, val)
                return obj

            return new

        new_error_class = type(class_name, (NVMLError,), {'__new__': gen_new(err_val)})
        new_error_class.__module__ = __name__
        setattr(this_module, class_name, new_error_class)
        NVMLError._valClassMapping[err_val] = new_error_class


_extractNVMLErrorsAsClasses()


def _nvmlCheckReturn(ret):
    if (ret != NVML_SUCCESS):
        raise NVMLError(ret)
    return ret


# Function access #
_nvmlGetFunctionPointer_cache = dict()  # function pointers are cached to prevent unnecessary libLoadLock locking


def _nvmlGetFunctionPointer(name):
    global nvmlLib

    if name in _nvmlGetFunctionPointer_cache:
        return _nvmlGetFunctionPointer_cache[name]

    libLoadLock.acquire()
    try:
        # ensure library was loaded
        if nvmlLib is None:
            raise NVMLError(NVML_ERROR_UNINITIALIZED)
        try:
            _nvmlGetFunctionPointer_cache[name] = getattr(nvmlLib, name)
            return _nvmlGetFunctionPointer_cache[name]
        except AttributeError:
            raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND)
    finally:
        # lock is always freed
        libLoadLock.release()


# Alternative object
# Allows the object to be printed
# Allows mismatched types to be assigned
#  - like None when the Structure variant requires c_uint
class nvmlFriendlyObject(object):
    def __init__(self, dictionary):
        for x in dictionary:
            setattr(self, x, dictionary[x])

    def __str__(self):
        return self.__dict__.__str__()


def nvmlStructToFriendlyObject(struct):
    d = {}
    for x in struct._fields_:
        key = x[0]
        value = getattr(struct, key)
        d[key] = value
    obj = nvmlFriendlyObject(d)
    return obj


# pack the object so it can be passed to the NVML library
def nvmlFriendlyObjectToStruct(obj, model):
    for x in model._fields_:
        key = x[0]
        value = obj.__dict__[key]
        setattr(model, key, value)
    return model


# Unit structures
class struct_c_nvmlUnit_t(Structure):
    pass  # opaque handle


c_nvmlUnit_t = POINTER(struct_c_nvmlUnit_t)


class _PrintableStructure(Structure):
    """
    Abstract class that produces nicer __str__ output than ctypes.Structure.
    e.g. instead of:
      >>> print str(obj)
      <class_name object at 0x7fdf82fef9e0>
    this class will print
      class_name(field_name: formatted_value, field_name: formatted_value)

    _fmt_ dictionary of <str _field_ name> -> <str format>
    e.g. class that has _field_ 'hex_value', c_uint could be formatted with
      _fmt_ = {"hex_value" : "%08X"}
    to produce nicer output.
    Default fomratting string for all fields can be set with key "<default>" like:
      _fmt_ = {"<default>" : "%d MHz"} # e.g all values are numbers in MHz.
    If not set it's assumed to be just "%s"

    Exact format of returned str from this class is subject to change in the future.
    """
    _fmt_ = {}

    def __str__(self):
        result = []
        for x in self._fields_:
            key = x[0]
            value = getattr(self, key)
            fmt = "%s"
            if key in self._fmt_:
                fmt = self._fmt_[key]
            elif "<default>" in self._fmt_:
                fmt = self._fmt_["<default>"]
            result.append(("%s: " + fmt) % (key, value))
        return self.__class__.__name__ + "(" + ", ".join(result) + ")"


class c_nvmlUnitInfo_t(_PrintableStructure):
    _fields_ = [
        ('name', c_char * 96),
        ('id', c_char * 96),
        ('serial', c_char * 96),
        ('firmwareVersion', c_char * 96),
    ]


class c_nvmlLedState_t(_PrintableStructure):
    _fields_ = [
        ('cause', c_char * 256),
        ('color', _nvmlLedColor_t),
    ]


class c_nvmlPSUInfo_t(_PrintableStructure):
    _fields_ = [
        ('state', c_char * 256),
        ('current', c_uint),
        ('voltage', c_uint),
        ('power', c_uint),
    ]


class c_nvmlUnitFanInfo_t(_PrintableStructure):
    _fields_ = [
        ('speed', c_uint),
        ('state', _nvmlFanState_t),
    ]


class c_nvmlUnitFanSpeeds_t(_PrintableStructure):
    _fields_ = [
        ('fans', c_nvmlUnitFanInfo_t * 24),
        ('count', c_uint)
    ]


# Device structures
class struct_c_nvmlDevice_t(Structure):
    pass  # opaque handle


c_nvmlDevice_t = POINTER(struct_c_nvmlDevice_t)


# Legacy pciInfo used for _v1 and _v2
class nvmlPciInfo_v2_t(_PrintableStructure):
    _fields_ = [
        ('busId', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE),
        ('domain', c_uint),
        ('bus', c_uint),
        ('device', c_uint),
        ('pciDeviceId', c_uint),

        # Added in 2.285
        ('pciSubSystemId', c_uint),
        ('reserved0', c_uint),
        ('reserved1', c_uint),
        ('reserved2', c_uint),
        ('reserved3', c_uint),
    ]
    _fmt_ = {
        'domain': "0x%04X",
        'bus': "0x%02X",
        'device': "0x%02X",
        'pciDeviceId': "0x%08X",
        'pciSubSystemId': "0x%08X",
    }


class nvmlPciInfo_t(_PrintableStructure):
    _fields_ = [
        # Moved to the new busId location below
        ('busIdLegacy', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE),
        ('domain', c_uint),
        ('bus', c_uint),
        ('device', c_uint),
        ('pciDeviceId', c_uint),

        # Added in 2.285
        ('pciSubSystemId', c_uint),
        # New busId replaced the long deprecated and reserved fields with a
        # field of the same size in 9.0
        ('busId', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE),
    ]
    _fmt_ = {
        'domain': "0x%08X",
        'bus': "0x%02X",
        'device': "0x%02X",
        'pciDeviceId': "0x%08X",
        'pciSubSystemId': "0x%08X",
    }


class c_nvmlBlacklistDeviceInfo_t(_PrintableStructure):
    _fields_ = [
        ('pci', nvmlPciInfo_t),
        ('uuid', c_char * NVML_DEVICE_UUID_BUFFER_SIZE)
    ]


class nvmlNvLinkUtilizationControl_t(_PrintableStructure):
    _fields_ = [
        ('units', _nvmlNvLinkUtilizationCountUnits_t),
        ('pktfilter', _nvmlNvLinkUtilizationCountPktTypes_t),
    ]


class c_nvmlMemory_t(_PrintableStructure):
    _fields_ = [
        ('total', c_ulonglong),
        ('free', c_ulonglong),
        ('used', c_ulonglong),
    ]
    _fmt_ = {'<default>': "%d B"}


class c_nvmlBAR1Memory_t(_PrintableStructure):
    _fields_ = [
        ('bar1Total', c_ulonglong),
        ('bar1Free', c_ulonglong),
        ('bar1Used', c_ulonglong),
    ]
    _fmt_ = {'<default>': "%d B"}


# On Windows with the WDDM driver, usedGpuMemory is reported as None
# Code that processes this structure should check for None, I.E.
#
# if (info.usedGpuMemory == None):
#     # TODO handle the error
#     pass
# else:
#    print("Using %d MiB of memory" % (info.usedGpuMemory / 1024 / 1024))
# endif
#
# See NVML documentation for more information
class c_nvmlProcessInfo_t(_PrintableStructure):
    _fields_ = [
        ('pid', c_uint),
        ('usedGpuMemory', c_ulonglong),
    ]
    _fmt_ = {'usedGpuMemory': "%d B"}


class c_nvmlBridgeChipInfo_t(_PrintableStructure):
    _fields_ = [
        ('type', _nvmlBridgeChipType_t),
        ('fwVersion', c_uint),
    ]


class c_nvmlBridgeChipHierarchy_t(_PrintableStructure):
    _fields_ = [
        ('bridgeCount', c_uint),
        ('bridgeChipInfo', c_nvmlBridgeChipInfo_t * 128),
    ]


class c_nvmlEccErrorCounts_t(_PrintableStructure):
    _fields_ = [
        ('l1Cache', c_ulonglong),
        ('l2Cache', c_ulonglong),
        ('deviceMemory', c_ulonglong),
        ('registerFile', c_ulonglong),
    ]


class c_nvmlUtilization_t(_PrintableStructure):
    _fields_ = [
        ('gpu', c_uint),
        ('memory', c_uint),
    ]
    _fmt_ = {'<default>': "%d %%"}


# Added in 2.285
class c_nvmlHwbcEntry_t(_PrintableStructure):
    _fields_ = [
        ('hwbcId', c_uint),
        ('firmwareVersion', c_char * 32),
    ]


class c_nvmlValue_t(Union):
    _fields_ = [
        ('dVal', c_double),
        ('uiVal', c_uint),
        ('ulVal', c_ulong),
        ('ullVal', c_ulonglong),
        ('sllVal', c_longlong),
    ]


class c_nvmlSample_t(_PrintableStructure):
    _fields_ = [
        ('timeStamp', c_ulonglong),
        ('sampleValue', c_nvmlValue_t),
    ]


class c_nvmlViolationTime_t(_PrintableStructure):
    _fields_ = [
        ('referenceTime', c_ulonglong),
        ('violationTime', c_ulonglong),
    ]


class c_nvmlFieldValue_t(_PrintableStructure):
    _fields_ = [
        ('fieldId', c_uint32),
        ('scopeId', c_uint32),
        ('timestamp', c_int64),
        ('latencyUsec', c_int64),
        ('valueType', _nvmlValueType_t),
        ('nvmlReturn', _nvmlReturn_t),
        ('value', c_nvmlValue_t)
    ]


class c_nvmlVgpuInstanceUtilizationSample_t(_PrintableStructure):
    _fields_ = [
        ('vgpuInstance', _nvmlVgpuInstance_t),
        ('timeStamp', c_ulonglong),
        ('smUtil', c_nvmlValue_t),
        ('memUtil', c_nvmlValue_t),
        ('encUtil', c_nvmlValue_t),
        ('decUtil', c_nvmlValue_t),
    ]


class c_nvmlVgpuProcessUtilizationSample_t(_PrintableStructure):
    _fields_ = [
        ('vgpuInstance', _nvmlVgpuInstance_t),
        ('pid', c_uint),
        ('processName', c_char * NVML_VGPU_NAME_BUFFER_SIZE),
        ('timeStamp', c_ulonglong),
        ('smUtil', c_uint),
        ('memUtil', c_uint),
        ('encUtil', c_uint),
        ('decUtil', c_uint),
    ]


class c_nvmlEncoderSession_t(_PrintableStructure):
    _fields_ = [
        ('sessionId', c_uint),
        ('pid', c_uint),
        ('vgpuInstance', _nvmlVgpuInstance_t),
        ('codecType', c_uint),
        ('hResolution', c_uint),
        ('vResolution', c_uint),
        ('averageFps', c_uint),
        ('encodeLatency', c_uint),
    ]


class c_nvmlProcessUtilizationSample_t(_PrintableStructure):
    _fields_ = [
        ('pid', c_uint),
        ('timeStamp', c_ulonglong),
        ('smUtil', c_uint),
        ('memUtil', c_uint),
        ('encUtil', c_uint),
        ('decUtil', c_uint),
    ]


class c_nvmlGridLicensableFeature_v3_t(_PrintableStructure):
    _fields_ = [
        ('featureCode', _nvmlGridLicenseFeatureCode_t),
        ('featureState', c_uint),
        ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
        ('productName', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
        ('featureEnabled', c_uint),
    ]


class c_nvmlGridLicensableFeatures_v3_t(_PrintableStructure):
    _fields_ = [
        ('isGridLicenseSupported', c_int),
        ('licensableFeaturesCount', c_uint),
        ('gridLicensableFeatures', c_nvmlGridLicensableFeature_v3_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT),
    ]


class c_nvmlGridLicensableFeature_v2_t(_PrintableStructure):
    _fields_ = [
        ('featureCode', _nvmlGridLicenseFeatureCode_t),
        ('featureState', c_uint),
        ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
        ('productName', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
    ]


class c_nvmlGridLicensableFeatures_v2_t(_PrintableStructure):
    _fields_ = [
        ('isGridLicenseSupported', c_int),
        ('licensableFeaturesCount', c_uint),
        ('gridLicensableFeatures', c_nvmlGridLicensableFeature_v2_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT),
    ]


class c_nvmlGridLicensableFeature_t(_PrintableStructure):
    _fields_ = [
        ('featureCode', _nvmlGridLicenseFeatureCode_t),
        ('featureState', c_uint),
        ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
    ]


class c_nvmlGridLicensableFeatures_t(_PrintableStructure):
    _fields_ = [
        ('isGridLicenseSupported', c_int),
        ('licensableFeaturesCount', c_uint),
        ('gridLicensableFeatures', c_nvmlGridLicensableFeature_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT),
    ]


## Event structures
class struct_c_nvmlEventSet_t(Structure):
    pass  # opaque handle


c_nvmlEventSet_t = POINTER(struct_c_nvmlEventSet_t)

nvmlEventTypeSingleBitEccError = 0x0000000000000001
nvmlEventTypeDoubleBitEccError = 0x0000000000000002
nvmlEventTypePState = 0x0000000000000004
nvmlEventTypeXidCriticalError = 0x0000000000000008
nvmlEventTypeClock = 0x0000000000000010
nvmlEventTypePowerSourceChange = 0x0000000000000080
nvmlEventMigConfigChange = 0x0000000000000100
nvmlEventTypeNone = 0x0000000000000000
nvmlEventTypeAll = (
        nvmlEventTypeNone
        | nvmlEventTypeSingleBitEccError
        | nvmlEventTypeDoubleBitEccError
        | nvmlEventTypePState
        | nvmlEventTypeClock
        | nvmlEventTypePowerSourceChange
        | nvmlEventTypeXidCriticalError
        | nvmlEventMigConfigChange
)

## Clock Throttle Reasons defines
nvmlClocksThrottleReasonGpuIdle = 0x0000000000000001
nvmlClocksThrottleReasonApplicationsClocksSetting = 0x0000000000000002
nvmlClocksThrottleReasonUserDefinedClocks = nvmlClocksThrottleReasonApplicationsClocksSetting  # deprecated,
# use nvmlClocksThrottleReasonApplicationsClocksSetting
nvmlClocksThrottleReasonSwPowerCap = 0x0000000000000004
nvmlClocksThrottleReasonHwSlowdown = 0x0000000000000008
nvmlClocksThrottleReasonSyncBoost = 0x0000000000000010
nvmlClocksThrottleReasonSwThermalSlowdown = 0x0000000000000020
nvmlClocksThrottleReasonHwThermalSlowdown = 0x0000000000000040
nvmlClocksThrottleReasonHwPowerBrakeSlowdown = 0x0000000000000080
nvmlClocksThrottleReasonDisplayClockSetting = 0x0000000000000100
nvmlClocksThrottleReasonNone = 0x0000000000000000
nvmlClocksThrottleReasonAll = (
        nvmlClocksThrottleReasonNone |
        nvmlClocksThrottleReasonGpuIdle |
        nvmlClocksThrottleReasonApplicationsClocksSetting |
        nvmlClocksThrottleReasonSwPowerCap |
        nvmlClocksThrottleReasonHwSlowdown |
        nvmlClocksThrottleReasonSyncBoost |
        nvmlClocksThrottleReasonSwThermalSlowdown |
        nvmlClocksThrottleReasonHwThermalSlowdown |
        nvmlClocksThrottleReasonHwPowerBrakeSlowdown |
        nvmlClocksThrottleReasonDisplayClockSetting
)


class c_nvmlEventData_t(_PrintableStructure):
    _fields_ = [
        ('device', c_nvmlDevice_t),
        ('eventType', c_ulonglong),
        ('eventData', c_ulonglong),
        ('gpuInstanceId', c_uint),
        ('computeInstanceId', c_uint)
    ]
    _fmt_ = {'eventType': "0x%08X"}


class c_nvmlAccountingStats_t(_PrintableStructure):
    _fields_ = [
        ('gpuUtilization', c_uint),
        ('memoryUtilization', c_uint),
        ('maxMemoryUsage', c_ulonglong),
        ('time', c_ulonglong),
        ('startTime', c_ulonglong),
        ('isRunning', c_uint),
        ('reserved', c_uint * 5)
    ]


class c_nvmlVgpuVersion_t(Structure):
    _fields_ = [("minVersion", c_uint),
                ("maxVersion", c_uint)
                ]


class c_nvmlVgpuMetadata_t(Structure):
    _fields_ = [("version", c_uint),
                ("revision", c_uint),
                ("guestInfoState", _nvmlVgpuGuestInfoState_t),
                ("guestDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
                ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
                ("reserved", c_uint * 6),
                ("vgpuVirtualizationCaps", c_uint),
                ("guestVgpuVersion", c_uint),
                ("opaqueDataSize", c_uint),
                ("opaqueData", c_char * NVML_VGPU_METADATA_OPAQUE_DATA_SIZE)
                ]


class c_nvmlVgpuPgpuMetadata_t(Structure):
    _fields_ = [("version", c_uint),
                ("revision", c_uint),
                ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
                ("pgpuVirtualizationCaps", c_uint),
                ("reserved", c_uint * 5),
                ("hostSupportedVgpuRange", c_nvmlVgpuVersion_t),
                ("opaqueDataSize", c_uint),
                ("opaqueData", c_char * NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE)
                ]


class c_nvmlVgpuPgpuCompatibility_t(Structure):
    _fields_ = [("vgpuVmCompatibility", _nvmlVgpuVmCompatibility_t),
                ("compatibilityLimitCode", _nvmlVgpuPgpuCompatibilityLimitCode_t)
                ]


class c_nvmlFBCStats_t(Structure):
    _fields_ = [("sessionsCount", c_uint),
                ("averageFPS", c_uint),
                ("averageLatency", c_uint)
                ]


class c_nvmlFBCSession_t(_PrintableStructure):
    _fields_ = [
        ('sessionId', c_uint),
        ('pid', c_uint),
        ('vgpuInstance', _nvmlVgpuInstance_t),
        ('displayOrdinal', c_uint),
        ('sessionType', c_uint),
        ('sessionFlags', c_uint),
        ('hMaxResolution', c_uint),
        ('vMaxResolution', c_uint),
        ('hResolution', c_uint),
        ('vResolution', c_uint),
        ('averageFPS', c_uint),
        ('averageLatency', c_uint),
    ]


NVML_DEVICE_MIG_DISABLE = 0x0
NVML_DEVICE_MIG_ENABLE = 0x1

NVML_GPU_INSTANCE_PROFILE_1_SLICE = 0x0
NVML_GPU_INSTANCE_PROFILE_2_SLICE = 0x1
NVML_GPU_INSTANCE_PROFILE_3_SLICE = 0x2
NVML_GPU_INSTANCE_PROFILE_4_SLICE = 0x3
NVML_GPU_INSTANCE_PROFILE_7_SLICE = 0x4
NVML_GPU_INSTANCE_PROFILE_COUNT = 0x5


class c_nvmlGpuInstancePlacement_t(Structure):
    _fields_ = [("start", c_uint),
                ("size", c_uint)
                ]


class c_nvmlGpuInstanceProfileInfo_t(Structure):
    _fields_ = [("id", c_uint),
                ("isP2pSupported", c_uint),
                ("sliceCount", c_uint),
                ("instanceCount", c_uint),
                ("multiprocessorCount", c_uint),
                ("copyEngineCount", c_uint),
                ("decoderCount", c_uint),
                ("encoderCount", c_uint),
                ("jpegCount", c_uint),
                ("ofaCount", c_uint),
                ("memorySizeMB", c_ulonglong),
                ]


class c_nvmlGpuInstanceInfo_t(Structure):
    _fields_ = [("device", c_nvmlDevice_t),
                ("id", c_uint),
                ("profileId", c_uint),
                ("placement", c_nvmlGpuInstancePlacement_t)
                ]


class struct_c_nvmlGpuInstance_t(Structure):
    pass  # opaque handle


c_nvmlGpuInstance_t = POINTER(struct_c_nvmlGpuInstance_t)

NVML_COMPUTE_INSTANCE_PROFILE_1_SLICE = 0x0
NVML_COMPUTE_INSTANCE_PROFILE_2_SLICE = 0x1
NVML_COMPUTE_INSTANCE_PROFILE_3_SLICE = 0x2
NVML_COMPUTE_INSTANCE_PROFILE_4_SLICE = 0x3
NVML_COMPUTE_INSTANCE_PROFILE_7_SLICE = 0x4
NVML_COMPUTE_INSTANCE_PROFILE_COUNT = 0x5

NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_SHARED = 0x0
NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT = 0x1


class c_nvmlComputeInstanceProfileInfo_t(Structure):
    _fields_ = [("id", c_uint),
                ("sliceCount", c_uint),
                ("instanceCount", c_uint),
                ("multiprocessorCount", c_uint),
                ("sharedCopyEngineCount", c_uint),
                ("sharedDecoderCount", c_uint),
                ("sharedEncoderCount", c_uint),
                ("sharedJpegCount", c_uint),
                ("sharedOfaCount", c_uint)
                ]


class c_nvmlComputeInstanceInfo_t(Structure):
    _fields_ = [("device", c_nvmlDevice_t),
                ("gpuInstance", c_nvmlGpuInstance_t),
                ("id", c_uint),
                ("profileId", c_uint)
                ]


class struct_c_nvmlComputeInstance_t(Structure):
    pass  # opaque handle


c_nvmlComputeInstance_t = POINTER(struct_c_nvmlComputeInstance_t)


class c_nvmlDeviceAttributes(Structure):
    _fields_ = [("multiprocessorCount", c_uint),
                ("sharedCopyEngineCount", c_uint),
                ("sharedDecoderCount", c_uint),
                ("sharedEncoderCount", c_uint),
                ("sharedJpegCount", c_uint),
                ("sharedOfaCount", c_uint),
                ("gpuInstanceSliceCount", c_uint),
                ("computeInstanceSliceCount", c_uint),
                ("memorySizeMB", c_ulonglong),
                ]


class c_nvmlRowRemapperHistogramValues(Structure):
    _fields_ = [("max", c_uint),
                ("high", c_uint),
                ("partial", c_uint),
                ("low", c_uint),
                ("none", c_uint)
                ]


## C function wrappers ##
def nvmlInitWithFlags(flags):
    _LoadNvmlLibrary()

    #
    # Initialize the library
    #
    fn = _nvmlGetFunctionPointer("nvmlInitWithFlags")
    ret = fn(flags)
    _nvmlCheckReturn(ret)

    # Atomically update refcount
    global _nvmlLib_refcount
    libLoadLock.acquire()
    _nvmlLib_refcount += 1
    libLoadLock.release()
    return None


def nvmlInit():
    nvmlInitWithFlags(0)
    return None


def _LoadNvmlLibrary():
    '''
    Load the library if it isn't loaded already
    '''
    global nvmlLib

    if nvmlLib is None:
        # lock to ensure only one caller loads the library
        libLoadLock.acquire()

        try:
            # ensure the library still isn't loaded
            if nvmlLib is None:
                try:
                    if (sys.platform[:3] == "win"):
                        searchPaths = [
                            os.path.join(os.getenv("ProgramFiles", r"C:\Program Files"),
                                         r"NVIDIA Corporation\NVSMI\nvml.dll"),
                            os.path.join(os.getenv("WinDir", r"C:\Windows"), r"System32\nvml.dll"),
                        ]
                        nvmlPath = next((x for x in searchPaths if os.path.isfile(x)), None)
                        if nvmlPath is None:
                            _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND)
                        else:
                            # cdecl calling convention
                            nvmlLib = CDLL(nvmlPath)
                    else:
                        # assume linux
                        nvmlLib = CDLL("libnvidia-ml.so.1")
                except OSError:
                    _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND)
                if nvmlLib is None:
                    _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND)
        finally:
            # lock is always freed
            libLoadLock.release()


def nvmlShutdown():
    #
    # Leave the library loaded, but shutdown the interface
    #
    fn = _nvmlGetFunctionPointer("nvmlShutdown")
    ret = fn()
    _nvmlCheckReturn(ret)

    # Atomically update refcount
    global _nvmlLib_refcount
    libLoadLock.acquire()
    if (0 < _nvmlLib_refcount):
        _nvmlLib_refcount -= 1
    libLoadLock.release()
    return None


# Added in 2.285
def nvmlErrorString(result):
    fn = _nvmlGetFunctionPointer("nvmlErrorString")
    fn.restype = c_char_p  # otherwise return is an int
    ret = fn(result)
    return ret


# Added in 2.285
def nvmlSystemGetNVMLVersion():
    c_version = create_string_buffer(NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlSystemGetNVMLVersion")
    ret = fn(c_version, c_uint(NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE))
    _nvmlCheckReturn(ret)
    return c_version.value


def nvmlSystemGetCudaDriverVersion():
    c_cuda_version = c_int()
    fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion")
    ret = fn(byref(c_cuda_version))
    _nvmlCheckReturn(ret)
    return c_cuda_version.value


def nvmlSystemGetCudaDriverVersion_v2():
    c_cuda_version = c_int()
    fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion_v2")
    ret = fn(byref(c_cuda_version))
    _nvmlCheckReturn(ret)
    return c_cuda_version.value


# Added in 2.285
def nvmlSystemGetProcessName(pid):
    c_name = create_string_buffer(1024)
    fn = _nvmlGetFunctionPointer("nvmlSystemGetProcessName")
    ret = fn(c_uint(pid), c_name, c_uint(1024))
    _nvmlCheckReturn(ret)
    return c_name.value


def nvmlSystemGetDriverVersion():
    c_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlSystemGetDriverVersion")
    ret = fn(c_version, c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE))
    _nvmlCheckReturn(ret)
    return c_version.value


# Added in 2.285
def nvmlSystemGetHicVersion():
    c_count = c_uint(0)
    hics = None
    fn = _nvmlGetFunctionPointer("nvmlSystemGetHicVersion")

    # get the count
    ret = fn(byref(c_count), None)

    # this should only fail with insufficient size
    if ((ret != NVML_SUCCESS) and
            (ret != NVML_ERROR_INSUFFICIENT_SIZE)):
        raise NVMLError(ret)

    # if there are no hics
    if (c_count.value == 0):
        return []

    hic_array = c_nvmlHwbcEntry_t * c_count.value
    hics = hic_array()
    ret = fn(byref(c_count), hics)
    _nvmlCheckReturn(ret)
    return hics


# Unit get functions
def nvmlUnitGetCount():
    c_count = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlUnitGetCount")
    ret = fn(byref(c_count))
    _nvmlCheckReturn(ret)
    return c_count.value


def nvmlUnitGetHandleByIndex(index):
    c_index = c_uint(index)
    unit = c_nvmlUnit_t()
    fn = _nvmlGetFunctionPointer("nvmlUnitGetHandleByIndex")
    ret = fn(c_index, byref(unit))
    _nvmlCheckReturn(ret)
    return unit


def nvmlUnitGetUnitInfo(unit):
    c_info = c_nvmlUnitInfo_t()
    fn = _nvmlGetFunctionPointer("nvmlUnitGetUnitInfo")
    ret = fn(unit, byref(c_info))
    _nvmlCheckReturn(ret)
    return c_info


def nvmlUnitGetLedState(unit):
    c_state = c_nvmlLedState_t()
    fn = _nvmlGetFunctionPointer("nvmlUnitGetLedState")
    ret = fn(unit, byref(c_state))
    _nvmlCheckReturn(ret)
    return c_state


def nvmlUnitGetPsuInfo(unit):
    c_info = c_nvmlPSUInfo_t()
    fn = _nvmlGetFunctionPointer("nvmlUnitGetPsuInfo")
    ret = fn(unit, byref(c_info))
    _nvmlCheckReturn(ret)
    return c_info


def nvmlUnitGetTemperature(unit, type):
    c_temp = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlUnitGetTemperature")
    ret = fn(unit, c_uint(type), byref(c_temp))
    _nvmlCheckReturn(ret)
    return c_temp.value


def nvmlUnitGetFanSpeedInfo(unit):
    c_speeds = c_nvmlUnitFanSpeeds_t()
    fn = _nvmlGetFunctionPointer("nvmlUnitGetFanSpeedInfo")
    ret = fn(unit, byref(c_speeds))
    _nvmlCheckReturn(ret)
    return c_speeds


# added to API
def nvmlUnitGetDeviceCount(unit):
    c_count = c_uint(0)
    # query the unit to determine device count
    fn = _nvmlGetFunctionPointer("nvmlUnitGetDevices")
    ret = fn(unit, byref(c_count), None)
    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        ret = NVML_SUCCESS
    _nvmlCheckReturn(ret)
    return c_count.value


def nvmlUnitGetDevices(unit):
    c_count = c_uint(nvmlUnitGetDeviceCount(unit))
    device_array = c_nvmlDevice_t * c_count.value
    c_devices = device_array()
    fn = _nvmlGetFunctionPointer("nvmlUnitGetDevices")
    ret = fn(unit, byref(c_count), c_devices)
    _nvmlCheckReturn(ret)
    return c_devices


# Device get functions
def nvmlDeviceGetCount():
    c_count = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCount_v2")
    ret = fn(byref(c_count))
    _nvmlCheckReturn(ret)
    return c_count.value


def nvmlDeviceGetHandleByIndex(index):
    c_index = c_uint(index)
    device = c_nvmlDevice_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByIndex_v2")
    ret = fn(c_index, byref(device))
    _nvmlCheckReturn(ret)
    return device


def nvmlDeviceGetHandleBySerial(serial):
    c_serial = c_char_p(serial)
    device = c_nvmlDevice_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleBySerial")
    ret = fn(c_serial, byref(device))
    _nvmlCheckReturn(ret)
    return device


def nvmlDeviceGetHandleByUUID(uuid):
    c_uuid = c_char_p(uuid)
    device = c_nvmlDevice_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByUUID")
    ret = fn(c_uuid, byref(device))
    _nvmlCheckReturn(ret)
    return device


def nvmlDeviceGetHandleByPciBusId(pciBusId):
    c_busId = c_char_p(pciBusId)
    device = c_nvmlDevice_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByPciBusId_v2")
    ret = fn(c_busId, byref(device))
    _nvmlCheckReturn(ret)
    return device


def nvmlDeviceGetName(handle):
    c_name = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetName")
    ret = fn(handle, c_name, c_uint(NVML_DEVICE_NAME_BUFFER_SIZE))
    _nvmlCheckReturn(ret)
    return c_name.value


def nvmlDeviceGetBoardId(handle):
    c_id = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBoardId")
    ret = fn(handle, byref(c_id))
    _nvmlCheckReturn(ret)
    return c_id.value


def nvmlDeviceGetMultiGpuBoard(handle):
    c_multiGpu = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMultiGpuBoard")
    ret = fn(handle, byref(c_multiGpu))
    _nvmlCheckReturn(ret)
    return c_multiGpu.value


def nvmlDeviceGetBrand(handle):
    c_type = _nvmlBrandType_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBrand")
    ret = fn(handle, byref(c_type))
    _nvmlCheckReturn(ret)
    return c_type.value


def nvmlDeviceGetBoardPartNumber(handle):
    c_part_number = create_string_buffer(NVML_DEVICE_PART_NUMBER_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBoardPartNumber")
    ret = fn(handle, c_part_number, c_uint(NVML_DEVICE_PART_NUMBER_BUFFER_SIZE))
    _nvmlCheckReturn(ret)
    return c_part_number.value


def nvmlDeviceGetSerial(handle):
    c_serial = create_string_buffer(NVML_DEVICE_SERIAL_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSerial")
    ret = fn(handle, c_serial, c_uint(NVML_DEVICE_SERIAL_BUFFER_SIZE))
    _nvmlCheckReturn(ret)
    return c_serial.value


def nvmlDeviceGetMemoryAffinity(handle, nodeSetSize, scope):
    affinity_array = c_ulonglong * nodeSetSize
    c_affinity = affinity_array()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryAffinity")
    ret = fn(handle, nodeSetSize, byref(c_affinity), _nvmlAffinityScope_t(scope))
    _nvmlCheckReturn(ret)
    return c_affinity


def nvmlDeviceGetCpuAffinityWithinScope(handle, cpuSetSize, scope):
    affinity_array = c_ulonglong * cpuSetSize
    c_affinity = affinity_array()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCpuAffinityWithinScope")
    ret = fn(handle, cpuSetSize, byref(c_affinity), _nvmlAffinityScope_t(scope))
    _nvmlCheckReturn(ret)
    return c_affinity


def nvmlDeviceGetCpuAffinity(handle, cpuSetSize):
    affinity_array = c_ulonglong * cpuSetSize
    c_affinity = affinity_array()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCpuAffinity")
    ret = fn(handle, cpuSetSize, byref(c_affinity))
    _nvmlCheckReturn(ret)
    return c_affinity


def nvmlDeviceSetCpuAffinity(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetCpuAffinity")
    ret = fn(handle)
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceClearCpuAffinity(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceClearCpuAffinity")
    ret = fn(handle)
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceGetMinorNumber(handle):
    c_minor_number = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinorNumber")
    ret = fn(handle, byref(c_minor_number))
    _nvmlCheckReturn(ret)
    return c_minor_number.value


def nvmlDeviceGetUUID(handle):
    c_uuid = create_string_buffer(NVML_DEVICE_UUID_V2_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetUUID")
    ret = fn(handle, c_uuid, c_uint(NVML_DEVICE_UUID_V2_BUFFER_SIZE))
    _nvmlCheckReturn(ret)
    return c_uuid.value


def nvmlDeviceGetInforomVersion(handle, infoRomObject):
    c_version = create_string_buffer(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomVersion")
    ret = fn(handle, _nvmlInforomObject_t(infoRomObject),
             c_version, c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE))
    _nvmlCheckReturn(ret)
    return c_version.value


# Added in 4.304
def nvmlDeviceGetInforomImageVersion(handle):
    c_version = create_string_buffer(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomImageVersion")
    ret = fn(handle, c_version, c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE))
    _nvmlCheckReturn(ret)
    return c_version.value


# Added in 4.304
def nvmlDeviceGetInforomConfigurationChecksum(handle):
    c_checksum = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomConfigurationChecksum")
    ret = fn(handle, byref(c_checksum))
    _nvmlCheckReturn(ret)
    return c_checksum.value


# Added in 4.304
def nvmlDeviceValidateInforom(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceValidateInforom")
    ret = fn(handle)
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceGetDisplayMode(handle):
    c_mode = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayMode")
    ret = fn(handle, byref(c_mode))
    _nvmlCheckReturn(ret)
    return c_mode.value


def nvmlDeviceGetDisplayActive(handle):
    c_mode = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayActive")
    ret = fn(handle, byref(c_mode))
    _nvmlCheckReturn(ret)
    return c_mode.value


def nvmlDeviceGetPersistenceMode(handle):
    c_state = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPersistenceMode")
    ret = fn(handle, byref(c_state))
    _nvmlCheckReturn(ret)
    return c_state.value


def nvmlDeviceGetPciInfo_v3(handle):
    c_info = nvmlPciInfo_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPciInfo_v3")
    ret = fn(handle, byref(c_info))
    _nvmlCheckReturn(ret)
    return c_info


def nvmlDeviceGetPciInfo(handle):
    return nvmlDeviceGetPciInfo_v3(handle)


def nvmlDeviceGetClockInfo(handle, type):
    c_clock = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetClockInfo")
    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
    _nvmlCheckReturn(ret)
    return c_clock.value


# Added in 2.285
def nvmlDeviceGetMaxClockInfo(handle, type):
    c_clock = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxClockInfo")
    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
    _nvmlCheckReturn(ret)
    return c_clock.value


# Added in 4.304
def nvmlDeviceGetApplicationsClock(handle, type):
    c_clock = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetApplicationsClock")
    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
    _nvmlCheckReturn(ret)
    return c_clock.value


def nvmlDeviceGetMaxCustomerBoostClock(handle, type):
    c_clock = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxCustomerBoostClock")
    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
    _nvmlCheckReturn(ret)
    return c_clock.value


def nvmlDeviceGetClock(handle, type, id):
    c_clock = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetClock")
    ret = fn(handle, _nvmlClockType_t(type), _nvmlClockId_t(id), byref(c_clock))
    _nvmlCheckReturn(ret)
    return c_clock.value


# Added in 5.319
def nvmlDeviceGetDefaultApplicationsClock(handle, type):
    c_clock = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDefaultApplicationsClock")
    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
    _nvmlCheckReturn(ret)
    return c_clock.value


# Added in 4.304
def nvmlDeviceGetSupportedMemoryClocks(handle):
    # first call to get the size
    c_count = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedMemoryClocks")
    ret = fn(handle, byref(c_count), None)

    if (ret == NVML_SUCCESS):
        # special case, no clocks
        return []
    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        clocks_array = c_uint * c_count.value
        c_clocks = clocks_array()

        # make the call again
        ret = fn(handle, byref(c_count), c_clocks)
        _nvmlCheckReturn(ret)

        procs = []
        for i in range(c_count.value):
            procs.append(c_clocks[i])

        return procs
    else:
        # error case
        raise NVMLError(ret)


# Added in 4.304
def nvmlDeviceGetSupportedGraphicsClocks(handle, memoryClockMHz):
    # first call to get the size
    c_count = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedGraphicsClocks")
    ret = fn(handle, c_uint(memoryClockMHz), byref(c_count), None)

    if (ret == NVML_SUCCESS):
        # special case, no clocks
        return []
    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        clocks_array = c_uint * c_count.value
        c_clocks = clocks_array()

        # make the call again
        ret = fn(handle, c_uint(memoryClockMHz), byref(c_count), c_clocks)
        _nvmlCheckReturn(ret)

        procs = []
        for i in range(c_count.value):
            procs.append(c_clocks[i])

        return procs
    else:
        # error case
        raise NVMLError(ret)


def nvmlDeviceGetFanSpeed(handle):
    c_speed = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed")
    ret = fn(handle, byref(c_speed))
    _nvmlCheckReturn(ret)
    return c_speed.value


def nvmlDeviceGetFanSpeed_v2(handle, fan):
    c_speed = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed_v2")
    ret = fn(handle, fan, byref(c_speed))
    _nvmlCheckReturn(ret)
    return c_speed.value


def nvmlDeviceGetTemperature(handle, sensor):
    c_temp = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperature")
    ret = fn(handle, _nvmlTemperatureSensors_t(sensor), byref(c_temp))
    _nvmlCheckReturn(ret)
    return c_temp.value


def nvmlDeviceGetTemperatureThreshold(handle, threshold):
    c_temp = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperatureThreshold")
    ret = fn(handle, _nvmlTemperatureThresholds_t(threshold), byref(c_temp))
    _nvmlCheckReturn(ret)
    return c_temp.value


# DEPRECATED use nvmlDeviceGetPerformanceState
def nvmlDeviceGetPowerState(handle):
    c_pstate = _nvmlPstates_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerState")
    ret = fn(handle, byref(c_pstate))
    _nvmlCheckReturn(ret)
    return c_pstate.value


def nvmlDeviceGetPerformanceState(handle):
    c_pstate = _nvmlPstates_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPerformanceState")
    ret = fn(handle, byref(c_pstate))
    _nvmlCheckReturn(ret)
    return c_pstate.value


def nvmlDeviceGetPowerManagementMode(handle):
    c_pcapMode = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementMode")
    ret = fn(handle, byref(c_pcapMode))
    _nvmlCheckReturn(ret)
    return c_pcapMode.value


def nvmlDeviceGetPowerManagementLimit(handle):
    c_limit = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementLimit")
    ret = fn(handle, byref(c_limit))
    _nvmlCheckReturn(ret)
    return c_limit.value


# Added in 4.304
def nvmlDeviceGetPowerManagementLimitConstraints(handle):
    c_minLimit = c_uint()
    c_maxLimit = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementLimitConstraints")
    ret = fn(handle, byref(c_minLimit), byref(c_maxLimit))
    _nvmlCheckReturn(ret)
    return [c_minLimit.value, c_maxLimit.value]


# Added in 4.304
def nvmlDeviceGetPowerManagementDefaultLimit(handle):
    c_limit = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementDefaultLimit")
    ret = fn(handle, byref(c_limit))
    _nvmlCheckReturn(ret)
    return c_limit.value


# Added in 331
def nvmlDeviceGetEnforcedPowerLimit(handle):
    c_limit = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEnforcedPowerLimit")
    ret = fn(handle, byref(c_limit))
    _nvmlCheckReturn(ret)
    return c_limit.value


def nvmlDeviceGetPowerUsage(handle):
    c_watts = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerUsage")
    ret = fn(handle, byref(c_watts))
    _nvmlCheckReturn(ret)
    return c_watts.value


def nvmlDeviceGetTotalEnergyConsumption(handle):
    c_millijoules = c_uint64()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEnergyConsumption")
    ret = fn(handle, byref(c_millijoules))
    _nvmlCheckReturn(ret)
    return c_millijoules.value


# Added in 4.304
def nvmlDeviceGetGpuOperationMode(handle):
    c_currState = _nvmlGpuOperationMode_t()
    c_pendingState = _nvmlGpuOperationMode_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuOperationMode")
    ret = fn(handle, byref(c_currState), byref(c_pendingState))
    _nvmlCheckReturn(ret)
    return [c_currState.value, c_pendingState.value]


# Added in 4.304
def nvmlDeviceGetCurrentGpuOperationMode(handle):
    return nvmlDeviceGetGpuOperationMode(handle)[0]


# Added in 4.304
def nvmlDeviceGetPendingGpuOperationMode(handle):
    return nvmlDeviceGetGpuOperationMode(handle)[1]


def nvmlDeviceGetMemoryInfo(handle):
    c_memory = c_nvmlMemory_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryInfo")
    ret = fn(handle, byref(c_memory))
    _nvmlCheckReturn(ret)
    return c_memory


def nvmlDeviceGetBAR1MemoryInfo(handle):
    c_bar1_memory = c_nvmlBAR1Memory_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBAR1MemoryInfo")
    ret = fn(handle, byref(c_bar1_memory))
    _nvmlCheckReturn(ret)
    return c_bar1_memory


def nvmlDeviceGetComputeMode(handle):
    c_mode = _nvmlComputeMode_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeMode")
    ret = fn(handle, byref(c_mode))
    _nvmlCheckReturn(ret)
    return c_mode.value


def nvmlDeviceGetCudaComputeCapability(handle):
    c_major = c_int()
    c_minor = c_int()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCudaComputeCapability")
    ret = fn(handle, byref(c_major), byref(c_minor))
    _nvmlCheckReturn(ret)
    return (c_major.value, c_minor.value)


def nvmlDeviceGetEccMode(handle):
    c_currState = _nvmlEnableState_t()
    c_pendingState = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEccMode")
    ret = fn(handle, byref(c_currState), byref(c_pendingState))
    _nvmlCheckReturn(ret)
    return [c_currState.value, c_pendingState.value]


# added to API
def nvmlDeviceGetCurrentEccMode(handle):
    return nvmlDeviceGetEccMode(handle)[0]


# added to API
def nvmlDeviceGetPendingEccMode(handle):
    return nvmlDeviceGetEccMode(handle)[1]


def nvmlDeviceGetTotalEccErrors(handle, errorType, counterType):
    c_count = c_ulonglong()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEccErrors")
    ret = fn(handle, _nvmlMemoryErrorType_t(errorType),
             _nvmlEccCounterType_t(counterType), byref(c_count))
    _nvmlCheckReturn(ret)
    return c_count.value


# This is deprecated, instead use nvmlDeviceGetMemoryErrorCounter
def nvmlDeviceGetDetailedEccErrors(handle, errorType, counterType):
    c_counts = c_nvmlEccErrorCounts_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDetailedEccErrors")
    ret = fn(handle, _nvmlMemoryErrorType_t(errorType),
             _nvmlEccCounterType_t(counterType), byref(c_counts))
    _nvmlCheckReturn(ret)
    return c_counts


# Added in 4.304
def nvmlDeviceGetMemoryErrorCounter(handle, errorType, counterType, locationType):
    c_count = c_ulonglong()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryErrorCounter")
    ret = fn(handle,
             _nvmlMemoryErrorType_t(errorType),
             _nvmlEccCounterType_t(counterType),
             _nvmlMemoryLocation_t(locationType),
             byref(c_count))
    _nvmlCheckReturn(ret)
    return c_count.value


def nvmlDeviceGetUtilizationRates(handle):
    c_util = c_nvmlUtilization_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetUtilizationRates")
    ret = fn(handle, byref(c_util))
    _nvmlCheckReturn(ret)
    return c_util


def nvmlDeviceGetEncoderUtilization(handle):
    c_util = c_uint()
    c_samplingPeriod = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderUtilization")
    ret = fn(handle, byref(c_util), byref(c_samplingPeriod))
    _nvmlCheckReturn(ret)
    return [c_util.value, c_samplingPeriod.value]


def nvmlDeviceGetDecoderUtilization(handle):
    c_util = c_uint()
    c_samplingPeriod = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDecoderUtilization")
    ret = fn(handle, byref(c_util), byref(c_samplingPeriod))
    _nvmlCheckReturn(ret)
    return [c_util.value, c_samplingPeriod.value]


def nvmlDeviceGetPcieReplayCounter(handle):
    c_replay = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieReplayCounter")
    ret = fn(handle, byref(c_replay))
    _nvmlCheckReturn(ret)
    return c_replay.value


def nvmlDeviceGetDriverModel(handle):
    c_currModel = _nvmlDriverModel_t()
    c_pendingModel = _nvmlDriverModel_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDriverModel")
    ret = fn(handle, byref(c_currModel), byref(c_pendingModel))
    _nvmlCheckReturn(ret)
    return [c_currModel.value, c_pendingModel.value]


# added to API
def nvmlDeviceGetCurrentDriverModel(handle):
    return nvmlDeviceGetDriverModel(handle)[0]


# added to API
def nvmlDeviceGetPendingDriverModel(handle):
    return nvmlDeviceGetDriverModel(handle)[1]


# Added in 2.285
def nvmlDeviceGetVbiosVersion(handle):
    c_version = create_string_buffer(NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVbiosVersion")
    ret = fn(handle, c_version, c_uint(NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE))
    _nvmlCheckReturn(ret)
    return c_version.value


# Added in 2.285
def nvmlDeviceGetComputeRunningProcesses(handle):
    # first call to get the size
    c_count = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeRunningProcesses")
    ret = fn(handle, byref(c_count), None)

    if (ret == NVML_SUCCESS):
        # special case, no running processes
        return []
    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        # oversize the array incase more processes are created
        c_count.value = c_count.value * 2 + 5
        proc_array = c_nvmlProcessInfo_t * c_count.value
        c_procs = proc_array()

        # make the call again
        ret = fn(handle, byref(c_count), c_procs)
        _nvmlCheckReturn(ret)

        procs = []
        for i in range(c_count.value):
            # use an alternative struct for this object
            obj = nvmlStructToFriendlyObject(c_procs[i])
            if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
                # special case for WDDM on Windows, see comment above
                obj.usedGpuMemory = None
            procs.append(obj)

        return procs
    else:
        # error case
        raise NVMLError(ret)


def nvmlDeviceGetGraphicsRunningProcesses(handle):
    # first call to get the size
    c_count = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGraphicsRunningProcesses")
    ret = fn(handle, byref(c_count), None)

    if (ret == NVML_SUCCESS):
        # special case, no running processes
        return []
    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        # oversize the array incase more processes are created
        c_count.value = c_count.value * 2 + 5
        proc_array = c_nvmlProcessInfo_t * c_count.value
        c_procs = proc_array()

        # make the call again
        ret = fn(handle, byref(c_count), c_procs)
        _nvmlCheckReturn(ret)

        procs = []
        for i in range(c_count.value):
            # use an alternative struct for this object
            obj = nvmlStructToFriendlyObject(c_procs[i])
            if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
                # special case for WDDM on Windows, see comment above
                obj.usedGpuMemory = None
            procs.append(obj)

        return procs
    else:
        # error case
        raise NVMLError(ret)


def nvmlDeviceGetAutoBoostedClocksEnabled(handle):
    c_isEnabled = _nvmlEnableState_t()
    c_defaultIsEnabled = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAutoBoostedClocksEnabled")
    ret = fn(handle, byref(c_isEnabled), byref(c_defaultIsEnabled))
    _nvmlCheckReturn(ret)
    return [c_isEnabled.value, c_defaultIsEnabled.value]
    # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks


## Set functions
def nvmlUnitSetLedState(unit, color):
    fn = _nvmlGetFunctionPointer("nvmlUnitSetLedState")
    ret = fn(unit, _nvmlLedColor_t(color))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceSetPersistenceMode(handle, mode):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetPersistenceMode")
    ret = fn(handle, _nvmlEnableState_t(mode))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceSetComputeMode(handle, mode):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetComputeMode")
    ret = fn(handle, _nvmlComputeMode_t(mode))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceSetEccMode(handle, mode):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetEccMode")
    ret = fn(handle, _nvmlEnableState_t(mode))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceClearEccErrorCounts(handle, counterType):
    fn = _nvmlGetFunctionPointer("nvmlDeviceClearEccErrorCounts")
    ret = fn(handle, _nvmlEccCounterType_t(counterType))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceSetDriverModel(handle, model):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetDriverModel")
    ret = fn(handle, _nvmlDriverModel_t(model))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceSetAutoBoostedClocksEnabled(handle, enabled):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetAutoBoostedClocksEnabled")
    ret = fn(handle, _nvmlEnableState_t(enabled))
    _nvmlCheckReturn(ret)
    return None
    # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks


def nvmlDeviceSetDefaultAutoBoostedClocksEnabled(handle, enabled, flags):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultAutoBoostedClocksEnabled")
    ret = fn(handle, _nvmlEnableState_t(enabled), c_uint(flags))
    _nvmlCheckReturn(ret)
    return None
    # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks


def nvmlDeviceSetGpuLockedClocks(handle, minGpuClockMHz, maxGpuClockMHz):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuLockedClocks")
    ret = fn(handle, c_uint(minGpuClockMHz), c_uint(maxGpuClockMHz))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceResetGpuLockedClocks(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceResetGpuLockedClocks")
    ret = fn(handle)
    _nvmlCheckReturn(ret)
    return None


# Added in 4.304
def nvmlDeviceSetApplicationsClocks(handle, maxMemClockMHz, maxGraphicsClockMHz):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetApplicationsClocks")
    ret = fn(handle, c_uint(maxMemClockMHz), c_uint(maxGraphicsClockMHz))
    _nvmlCheckReturn(ret)
    return None


# Added in 4.304
def nvmlDeviceResetApplicationsClocks(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceResetApplicationsClocks")
    ret = fn(handle)
    _nvmlCheckReturn(ret)
    return None


# Added in 4.304
def nvmlDeviceSetPowerManagementLimit(handle, limit):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetPowerManagementLimit")
    ret = fn(handle, c_uint(limit))
    _nvmlCheckReturn(ret)
    return None


# Added in 4.304
def nvmlDeviceSetGpuOperationMode(handle, mode):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuOperationMode")
    ret = fn(handle, _nvmlGpuOperationMode_t(mode))
    _nvmlCheckReturn(ret)
    return None


# Added in 2.285
def nvmlEventSetCreate():
    fn = _nvmlGetFunctionPointer("nvmlEventSetCreate")
    eventSet = c_nvmlEventSet_t()
    ret = fn(byref(eventSet))
    _nvmlCheckReturn(ret)
    return eventSet


# Added in 2.285
def nvmlDeviceRegisterEvents(handle, eventTypes, eventSet):
    fn = _nvmlGetFunctionPointer("nvmlDeviceRegisterEvents")
    ret = fn(handle, c_ulonglong(eventTypes), eventSet)
    _nvmlCheckReturn(ret)
    return None


# Added in 2.285
def nvmlDeviceGetSupportedEventTypes(handle):
    c_eventTypes = c_ulonglong()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedEventTypes")
    ret = fn(handle, byref(c_eventTypes))
    _nvmlCheckReturn(ret)
    return c_eventTypes.value


# raises NVML_ERROR_TIMEOUT exception on timeout
def nvmlEventSetWait_v2(eventSet, timeoutms):
    fn = _nvmlGetFunctionPointer("nvmlEventSetWait_v2")
    data = c_nvmlEventData_t()
    ret = fn(eventSet, byref(data), c_uint(timeoutms))
    _nvmlCheckReturn(ret)
    return data


def nvmlEventSetWait(eventSet, timeoutms):
    return nvmlEventSetWait_v2(eventSet, timeoutms)


# Added in 2.285
def nvmlEventSetFree(eventSet):
    fn = _nvmlGetFunctionPointer("nvmlEventSetFree")
    ret = fn(eventSet)
    _nvmlCheckReturn(ret)
    return None


# Added in 3.295
def nvmlDeviceOnSameBoard(handle1, handle2):
    fn = _nvmlGetFunctionPointer("nvmlDeviceOnSameBoard")
    onSameBoard = c_int()
    ret = fn(handle1, handle2, byref(onSameBoard))
    _nvmlCheckReturn(ret)
    return (onSameBoard.value != 0)


# Added in 3.295
def nvmlDeviceGetCurrPcieLinkGeneration(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrPcieLinkGeneration")
    gen = c_uint()
    ret = fn(handle, byref(gen))
    _nvmlCheckReturn(ret)
    return gen.value


# Added in 3.295
def nvmlDeviceGetMaxPcieLinkGeneration(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkGeneration")
    gen = c_uint()
    ret = fn(handle, byref(gen))
    _nvmlCheckReturn(ret)
    return gen.value


# Added in 3.295
def nvmlDeviceGetCurrPcieLinkWidth(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrPcieLinkWidth")
    width = c_uint()
    ret = fn(handle, byref(width))
    _nvmlCheckReturn(ret)
    return width.value


# Added in 3.295
def nvmlDeviceGetMaxPcieLinkWidth(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkWidth")
    width = c_uint()
    ret = fn(handle, byref(width))
    _nvmlCheckReturn(ret)
    return width.value


# Added in 4.304
def nvmlDeviceGetSupportedClocksThrottleReasons(handle):
    c_reasons = c_ulonglong()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedClocksThrottleReasons")
    ret = fn(handle, byref(c_reasons))
    _nvmlCheckReturn(ret)
    return c_reasons.value


# Added in 4.304
def nvmlDeviceGetCurrentClocksThrottleReasons(handle):
    c_reasons = c_ulonglong()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClocksThrottleReasons")
    ret = fn(handle, byref(c_reasons))
    _nvmlCheckReturn(ret)
    return c_reasons.value


# Added in 5.319
def nvmlDeviceGetIndex(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetIndex")
    c_index = c_uint()
    ret = fn(handle, byref(c_index))
    _nvmlCheckReturn(ret)
    return c_index.value


# Added in 5.319
def nvmlDeviceGetAccountingMode(handle):
    c_mode = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingMode")
    ret = fn(handle, byref(c_mode))
    _nvmlCheckReturn(ret)
    return c_mode.value


def nvmlDeviceSetAccountingMode(handle, mode):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetAccountingMode")
    ret = fn(handle, _nvmlEnableState_t(mode))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceClearAccountingPids(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceClearAccountingPids")
    ret = fn(handle)
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceGetAccountingStats(handle, pid):
    stats = c_nvmlAccountingStats_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingStats")
    ret = fn(handle, c_uint(pid), byref(stats))
    _nvmlCheckReturn(ret)
    if (stats.maxMemoryUsage == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
        # special case for WDDM on Windows, see comment above
        stats.maxMemoryUsage = None
    return stats


def nvmlDeviceGetAccountingPids(handle):
    count = c_uint(nvmlDeviceGetAccountingBufferSize(handle))
    pids = (c_uint * count.value)()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingPids")
    ret = fn(handle, byref(count), pids)
    _nvmlCheckReturn(ret)
    return list(map(int, pids[0:count.value]))


def nvmlDeviceGetAccountingBufferSize(handle):
    bufferSize = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingBufferSize")
    ret = fn(handle, byref(bufferSize))
    _nvmlCheckReturn(ret)
    return int(bufferSize.value)


def nvmlDeviceGetRetiredPages(device, sourceFilter):
    c_source = _nvmlPageRetirementCause_t(sourceFilter)
    c_count = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPages")

    # First call will get the size
    ret = fn(device, c_source, byref(c_count), None)

    # this should only fail with insufficient size
    if ((ret != NVML_SUCCESS) and
            (ret != NVML_ERROR_INSUFFICIENT_SIZE)):
        raise NVMLError(ret)

    # call again with a buffer
    # oversize the array for the rare cases where additional pages
    # are retired between NVML calls
    c_count.value = c_count.value * 2 + 5
    page_array = c_ulonglong * c_count.value
    c_pages = page_array()
    ret = fn(device, c_source, byref(c_count), c_pages)
    _nvmlCheckReturn(ret)
    return list(map(int, c_pages[0:c_count.value]))


def nvmlDeviceGetRetiredPages_v2(device, sourceFilter):
    c_source = _nvmlPageRetirementCause_t(sourceFilter)
    c_count = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPages_v2")

    # First call will get the size
    ret = fn(device, c_source, byref(c_count), None)

    # this should only fail with insufficient size
    if ((ret != NVML_SUCCESS) and
            (ret != NVML_ERROR_INSUFFICIENT_SIZE)):
        raise NVMLError(ret)

    # call again with a buffer
    # oversize the array for the rare cases where additional pages
    # are retired between NVML calls
    c_count.value = c_count.value * 2 + 5
    page_array = c_ulonglong * c_count.value
    c_pages = page_array()
    times_array = c_ulonglong * c_count.value
    c_times = times_array()
    ret = fn(device, c_source, byref(c_count), c_pages, c_times)
    _nvmlCheckReturn(ret)
    return [{'address': int(c_pages[i]), 'timestamp': int(c_times[i])} for i in range(c_count.value)];


def nvmlDeviceGetRetiredPagesPendingStatus(device):
    c_pending = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPagesPendingStatus")
    ret = fn(device, byref(c_pending))
    _nvmlCheckReturn(ret)
    return int(c_pending.value)


def nvmlDeviceGetAPIRestriction(device, apiType):
    c_permission = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAPIRestriction")
    ret = fn(device, _nvmlRestrictedAPI_t(apiType), byref(c_permission))
    _nvmlCheckReturn(ret)
    return int(c_permission.value)


def nvmlDeviceSetAPIRestriction(handle, apiType, isRestricted):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetAPIRestriction")
    ret = fn(handle, _nvmlRestrictedAPI_t(apiType), _nvmlEnableState_t(isRestricted))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceGetBridgeChipInfo(handle):
    bridgeHierarchy = c_nvmlBridgeChipHierarchy_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBridgeChipInfo")
    ret = fn(handle, byref(bridgeHierarchy))
    _nvmlCheckReturn(ret)
    return bridgeHierarchy


def nvmlDeviceGetSamples(device, sampling_type, timeStamp):
    c_sampling_type = _nvmlSamplingType_t(sampling_type)
    c_time_stamp = c_ulonglong(timeStamp)
    c_sample_count = c_uint(0)
    c_sample_value_type = _nvmlValueType_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSamples")

    ## First Call gets the size
    ret = fn(device, c_sampling_type, c_time_stamp, byref(c_sample_value_type), byref(c_sample_count), None)

    # Stop if this fails
    if (ret != NVML_SUCCESS):
        raise NVMLError(ret)

    sampleArray = c_sample_count.value * c_nvmlSample_t
    c_samples = sampleArray()
    ret = fn(device, c_sampling_type, c_time_stamp, byref(c_sample_value_type), byref(c_sample_count), c_samples)
    _nvmlCheckReturn(ret)
    return (c_sample_value_type.value, c_samples[0:c_sample_count.value])


def nvmlDeviceGetViolationStatus(device, perfPolicyType):
    c_perfPolicy_type = _nvmlPerfPolicyType_t(perfPolicyType)
    c_violTime = c_nvmlViolationTime_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetViolationStatus")

    ## Invoke the method to get violation time
    ret = fn(device, c_perfPolicy_type, byref(c_violTime))
    _nvmlCheckReturn(ret)
    return c_violTime


def nvmlDeviceGetPcieThroughput(device, counter):
    c_util = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieThroughput")
    ret = fn(device, _nvmlPcieUtilCounter_t(counter), byref(c_util))
    _nvmlCheckReturn(ret)
    return c_util.value


def nvmlSystemGetTopologyGpuSet(cpuNumber):
    c_count = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlSystemGetTopologyGpuSet")

    # First call will get the size
    ret = fn(cpuNumber, byref(c_count), None)

    if ret != NVML_SUCCESS:
        raise NVMLError(ret)
    # call again with a buffer
    device_array = c_nvmlDevice_t * c_count.value
    c_devices = device_array()
    ret = fn(cpuNumber, byref(c_count), c_devices)
    _nvmlCheckReturn(ret)
    return list(c_devices[0:c_count.value])


def nvmlDeviceGetTopologyNearestGpus(device, level):
    c_count = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTopologyNearestGpus")

    # First call will get the size
    ret = fn(device, level, byref(c_count), None)

    if ret != NVML_SUCCESS:
        raise NVMLError(ret)

    # call again with a buffer
    device_array = c_nvmlDevice_t * c_count.value
    c_devices = device_array()
    ret = fn(device, level, byref(c_count), c_devices)
    _nvmlCheckReturn(ret)
    return list(c_devices[0:c_count.value])


def nvmlDeviceGetTopologyCommonAncestor(device1, device2):
    c_level = _nvmlGpuTopologyLevel_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTopologyCommonAncestor")
    ret = fn(device1, device2, byref(c_level))
    _nvmlCheckReturn(ret)
    return c_level.value


def nvmlDeviceGetNvLinkUtilizationCounter(device, link, counter):
    c_rxcounter = c_ulonglong()
    c_txcounter = c_ulonglong()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkUtilizationCounter")
    ret = fn(device, link, counter, byref(c_rxcounter), byref(c_txcounter))
    _nvmlCheckReturn(ret)
    return (c_rxcounter.value, c_txcounter.value)


def nvmlDeviceFreezeNvLinkUtilizationCounter(device, link, counter, freeze):
    fn = _nvmlGetFunctionPointer("nvmlDeviceFreezeNvLinkUtilizationCounter")
    ret = fn(device, link, counter, freeze)
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceResetNvLinkUtilizationCounter(device, link, counter):
    fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkUtilizationCounter")
    ret = fn(device, link, counter)
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceSetNvLinkUtilizationControl(device, link, counter, control, reset):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvLinkUtilizationControl")
    ret = fn(device, link, counter, byref(control), reset)
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceGetNvLinkUtilizationControl(device, link, counter):
    c_control = nvmlNvLinkUtilizationControl_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkUtilizationControl")
    ret = fn(device, link, counter, byref(c_control))
    _nvmlCheckReturn(ret)
    return c_control


def nvmlDeviceGetNvLinkCapability(device, link, capability):
    c_capResult = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkCapability")
    ret = fn(device, link, capability, byref(c_capResult))
    _nvmlCheckReturn(ret)
    return c_capResult.value


def nvmlDeviceGetNvLinkErrorCounter(device, link, counter):
    c_result = c_ulonglong()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkErrorCounter")
    ret = fn(device, link, counter, byref(c_result))
    _nvmlCheckReturn(ret)
    return c_result.value


def nvmlDeviceResetNvLinkErrorCounters(device, link):
    fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkErrorCounters")
    ret = fn(device, link)
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceGetNvLinkRemotePciInfo(device, link):
    c_pci = nvmlPciInfo_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkRemotePciInfo_v2")
    ret = fn(device, link, byref(c_pci))
    _nvmlCheckReturn(ret)
    return c_pci


def nvmlDeviceGetNvLinkState(device, link):
    c_isActive = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkState")
    ret = fn(device, link, byref(c_isActive))
    _nvmlCheckReturn(ret)
    return c_isActive.value


def nvmlDeviceGetNvLinkVersion(device, link):
    c_version = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkVersion")
    ret = fn(device, link, byref(c_version))
    _nvmlCheckReturn(ret)
    return c_version.value


def nvmlDeviceModifyDrainState(pciInfo, newState):
    fn = _nvmlGetFunctionPointer("nvmlDeviceModifyDrainState")
    ret = fn(pointer(pciInfo), newState)
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceQueryDrainState(pciInfo):
    c_newState = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceQueryDrainState")
    ret = fn(pointer(pciInfo), byref(c_newState))
    _nvmlCheckReturn(ret)
    return c_newState.value


def nvmlDeviceRemoveGpu(pciInfo):
    fn = _nvmlGetFunctionPointer("nvmlDeviceRemoveGpu")
    ret = fn(pointer(pciInfo))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceDiscoverGpus(pciInfo):
    fn = _nvmlGetFunctionPointer("nvmlDeviceDiscoverGpus")
    ret = fn(pointer(pciInfo))
    _nvmlCheckReturn(ret)
    return None


def nvmlDeviceGetFieldValues(handle, fieldIds):
    values_arr = c_nvmlFieldValue_t * len(fieldIds)
    values = values_arr()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFieldValues")

    for i, fieldId in enumerate(fieldIds):
        try:
            (values[i].fieldId, values[i].scopeId) = fieldId
        except TypeError:
            values[i].fieldId = fieldId

    ret = fn(handle, c_int32(len(fieldIds)), byref(values))
    _nvmlCheckReturn(ret)
    return values


def nvmlDeviceGetVirtualizationMode(handle):
    c_virtualization_mode = c_ulonglong()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVirtualizationMode")
    ret = fn(handle, byref(c_virtualization_mode))
    _nvmlCheckReturn(ret)
    return c_virtualization_mode.value


def nvmlDeviceSetVirtualizationMode(handle, virtualization_mode):
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetVirtualizationMode")
    return fn(handle, virtualization_mode)


def nvmlDeviceGetSupportedVgpus(handle):
    # first call to get the size
    c_vgpu_count = c_uint(0)

    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedVgpus")
    ret = fn(handle, byref(c_vgpu_count), None)

    if (ret == NVML_SUCCESS):
        # special case, no supported vGPUs
        return []
    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value
        c_vgpu_type_ids = vgpu_type_ids_array()

        # make the call again
        ret = fn(handle, byref(c_vgpu_count), c_vgpu_type_ids)
        _nvmlCheckReturn(ret)
        vgpus = []
        for i in range(c_vgpu_count.value):
            vgpus.append(c_vgpu_type_ids[i])
        return vgpus
    else:
        # error case
        raise NVMLError(ret)


def nvmlDeviceGetCreatableVgpus(handle):
    # first call to get the size
    c_vgpu_count = c_uint(0)

    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCreatableVgpus")
    ret = fn(handle, byref(c_vgpu_count), None)

    if (ret == NVML_SUCCESS):
        # special case, no supported vGPUs
        return []
    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value
        c_vgpu_type_ids = vgpu_type_ids_array()

        # make the call again
        ret = fn(handle, byref(c_vgpu_count), c_vgpu_type_ids)
        _nvmlCheckReturn(ret)
        vgpus = []
        for i in range(c_vgpu_count.value):
            vgpus.append(c_vgpu_type_ids[i])
        return vgpus
    else:
        # error case
        raise NVMLError(ret)


def nvmlVgpuTypeGetClass(vgpuTypeId):
    c_class = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE)
    c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetClass")
    ret = fn(vgpuTypeId, c_class, byref(c_buffer_size))
    _nvmlCheckReturn(ret)
    return c_class.value


def nvmlVgpuTypeGetName(vgpuTypeId):
    c_name = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE)
    c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetName")
    ret = fn(vgpuTypeId, c_name, byref(c_buffer_size))
    _nvmlCheckReturn(ret)
    return c_name.value


def nvmlVgpuTypeGetDeviceID(vgpuTypeId):
    c_device_id = c_ulonglong(0)
    c_subsystem_id = c_ulonglong(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetDeviceID")
    ret = fn(vgpuTypeId, byref(c_device_id), byref(c_subsystem_id))
    _nvmlCheckReturn(ret)
    return (c_device_id.value, c_subsystem_id.value)


def nvmlVgpuTypeGetFramebufferSize(vgpuTypeId):
    c_fb_size = c_ulonglong(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFramebufferSize")
    ret = fn(vgpuTypeId, byref(c_fb_size))
    _nvmlCheckReturn(ret)
    return c_fb_size.value


def nvmlVgpuTypeGetNumDisplayHeads(vgpuTypeId):
    c_num_heads = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetNumDisplayHeads")
    ret = fn(vgpuTypeId, byref(c_num_heads))
    _nvmlCheckReturn(ret)
    return c_num_heads.value


def nvmlVgpuTypeGetResolution(vgpuTypeId):
    c_xdim = c_uint(0)
    c_ydim = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetResolution")
    ret = fn(vgpuTypeId, 0, byref(c_xdim), byref(c_ydim))
    _nvmlCheckReturn(ret)
    return (c_xdim.value, c_ydim.value)


def nvmlVgpuTypeGetLicense(vgpuTypeId):
    c_license = create_string_buffer(NVML_GRID_LICENSE_BUFFER_SIZE)
    c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetLicense")
    ret = fn(vgpuTypeId, c_license, c_buffer_size)
    _nvmlCheckReturn(ret)
    return c_license.value


def nvmlVgpuTypeGetFrameRateLimit(vgpuTypeId):
    c_frl_config = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFrameRateLimit")
    ret = fn(vgpuTypeId, byref(c_frl_config))
    _nvmlCheckReturn(ret)
    return c_frl_config.value


def nvmlVgpuTypeGetMaxInstances(handle, vgpuTypeId):
    c_max_instances = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances")
    ret = fn(handle, vgpuTypeId, byref(c_max_instances))
    _nvmlCheckReturn(ret)
    return c_max_instances.value


def nvmlVgpuTypeGetMaxInstancesPerVm(vgpuTypeId):
    c_max_instances_per_vm = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstancesPerVm")
    ret = fn(vgpuTypeId, byref(c_max_instances_per_vm))
    _nvmlCheckReturn(ret)
    return c_max_instances_per_vm.value


def nvmlDeviceGetActiveVgpus(handle):
    # first call to get the size
    c_vgpu_count = c_uint(0)

    fn = _nvmlGetFunctionPointer("nvmlDeviceGetActiveVgpus")
    ret = fn(handle, byref(c_vgpu_count), None)

    if (ret == NVML_SUCCESS):
        # special case, no active vGPUs
        return []
    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        vgpu_instance_array = _nvmlVgpuInstance_t * c_vgpu_count.value
        c_vgpu_instances = vgpu_instance_array()

        # make the call again
        ret = fn(handle, byref(c_vgpu_count), c_vgpu_instances)
        _nvmlCheckReturn(ret)
        vgpus = []
        for i in range(c_vgpu_count.value):
            vgpus.append(c_vgpu_instances[i])
        return vgpus
    else:
        # error case
        raise NVMLError(ret)


def nvmlVgpuInstanceGetVmID(vgpuInstance):
    c_vm_id = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE)
    c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE)
    c_vm_id_type = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmID")
    ret = fn(vgpuInstance, byref(c_vm_id), c_buffer_size, byref(c_vm_id_type))
    _nvmlCheckReturn(ret)
    return (c_vm_id.value, c_vm_id_type.value)


def nvmlVgpuInstanceGetUUID(vgpuInstance):
    c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE)
    c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetUUID")
    ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size)
    _nvmlCheckReturn(ret)
    return c_uuid.value


def nvmlVgpuInstanceGetMdevUUID(vgpuInstance):
    c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE)
    c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMdevUUID")
    ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size)
    _nvmlCheckReturn(ret)
    return c_uuid.value


def nvmlVgpuInstanceGetVmDriverVersion(vgpuInstance):
    c_driver_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)
    c_buffer_size = c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmDriverVersion")
    ret = fn(vgpuInstance, byref(c_driver_version), c_buffer_size)
    _nvmlCheckReturn(ret)
    return c_driver_version.value


def nvmlVgpuInstanceGetLicenseStatus(vgpuInstance):
    c_license_status = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseStatus")
    ret = fn(vgpuInstance, byref(c_license_status))
    _nvmlCheckReturn(ret)
    return c_license_status.value


def nvmlVgpuInstanceGetFrameRateLimit(vgpuInstance):
    c_frl = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFrameRateLimit")
    ret = fn(vgpuInstance, byref(c_frl))
    _nvmlCheckReturn(ret)
    return c_frl.value


def nvmlVgpuInstanceGetEccMode(vgpuInstance):
    c_mode = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEccMode")
    ret = fn(vgpuInstance, byref(c_mode))
    _nvmlCheckReturn(ret)
    return c_mode.value


def nvmlVgpuInstanceGetType(vgpuInstance):
    c_vgpu_type = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetType")
    ret = fn(vgpuInstance, byref(c_vgpu_type))
    _nvmlCheckReturn(ret)
    return c_vgpu_type.value


def nvmlVgpuInstanceGetEncoderCapacity(vgpuInstance):
    c_encoder_capacity = c_ulonglong(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderCapacity")
    ret = fn(vgpuInstance, byref(c_encoder_capacity))
    _nvmlCheckReturn(ret)
    return c_encoder_capacity.value


def nvmlVgpuInstanceSetEncoderCapacity(vgpuInstance, encoder_capacity):
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceSetEncoderCapacity")
    return fn(vgpuInstance, encoder_capacity)


def nvmlVgpuInstanceGetFbUsage(vgpuInstance):
    c_fb_usage = c_uint(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFbUsage")
    ret = fn(vgpuInstance, byref(c_fb_usage))
    _nvmlCheckReturn(ret)
    return c_fb_usage.value


def nvmlDeviceGetVgpuUtilization(handle, timeStamp):
    # first call to get the size
    c_vgpu_count = c_uint(0)
    c_time_stamp = c_ulonglong(timeStamp)
    c_sample_value_type = _nvmlValueType_t()

    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuUtilization")
    ret = fn(handle, c_time_stamp, byref(c_sample_value_type), byref(c_vgpu_count), None)

    if (ret == NVML_SUCCESS):
        # special case, no active vGPUs
        return []
    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        sampleArray = c_vgpu_count.value * c_nvmlVgpuInstanceUtilizationSample_t
        c_samples = sampleArray()

        # make the call again
        ret = fn(handle, c_time_stamp, byref(c_sample_value_type), byref(c_vgpu_count), c_samples)
        _nvmlCheckReturn(ret)

        return c_samples[0:c_vgpu_count.value]
    else:
        # error case
        raise NVMLError(ret)


def nvmlDeviceGetP2PStatus(device1, device2, p2pIndex):
    c_p2pstatus = _nvmlGpuP2PStatus_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetP2PStatus")
    ret = fn(device1, device2, p2pIndex, byref(c_p2pstatus))
    _nvmlCheckReturn(ret)
    return c_p2pstatus.value


def nvmlDeviceGetGridLicensableFeatures_v3(handle):
    c_get_grid_licensable_features = c_nvmlGridLicensableFeatures_v3_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGridLicensableFeatures_v3")
    ret = fn(handle, byref(c_get_grid_licensable_features))
    _nvmlCheckReturn(ret)

    return (c_get_grid_licensable_features)


def nvmlDeviceGetGridLicensableFeatures(handle):
    return nvmlDeviceGetGridLicensableFeatures_v3(handle)


def nvmlDeviceGetEncoderCapacity(handle, encoderQueryType):
    c_encoder_capacity = c_ulonglong(0)
    c_encoderQuery_type = _nvmlEncoderQueryType_t(encoderQueryType)

    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderCapacity")
    ret = fn(handle, c_encoderQuery_type, byref(c_encoder_capacity))
    _nvmlCheckReturn(ret)
    return c_encoder_capacity.value


def nvmlDeviceGetVgpuProcessUtilization(handle, timeStamp):
    # first call to get the size
    c_vgpu_count = c_uint(0)
    c_time_stamp = c_ulonglong(timeStamp)

    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessUtilization")
    ret = fn(handle, c_time_stamp, byref(c_vgpu_count), None)

    if (ret == NVML_SUCCESS):
        # special case, no active vGPUs
        return []
    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        sampleArray = c_vgpu_count.value * c_nvmlVgpuProcessUtilizationSample_t
        c_samples = sampleArray()

        # make the call again
        ret = fn(handle, c_time_stamp, byref(c_vgpu_count), c_samples)
        _nvmlCheckReturn(ret)

        return c_samples[0:c_vgpu_count.value]
    else:
        # error case
        raise NVMLError(ret)


def nvmlDeviceGetEncoderStats(handle):
    c_encoderCount = c_ulonglong(0)
    c_encodeFps = c_ulonglong(0)
    c_encoderLatency = c_ulonglong(0)
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderStats")
    ret = fn(handle, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency))
    _nvmlCheckReturn(ret)
    return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value)


def nvmlDeviceGetEncoderSessions(handle):
    # first call to get the size
    c_session_count = c_uint(0)

    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderSessions")
    ret = fn(handle, byref(c_session_count), None)

    if (ret == NVML_SUCCESS):
        if (c_session_count.value != 0):
            # typical case
            session_array = c_nvmlEncoderSession_t * c_session_count.value
            c_sessions = session_array()

            # make the call again
            ret = fn(handle, byref(c_session_count), c_sessions)
            _nvmlCheckReturn(ret)
            sessions = []
            for i in range(c_session_count.value):
                sessions.append(c_sessions[i])
            return sessions
        else:
            return []  # no active sessions
    else:
        # error case
        raise NVMLError(ret)


def nvmlDeviceGetFBCStats(handle):
    c_fbcStats = c_nvmlFBCStats_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCStats")
    ret = fn(handle, byref(c_fbcStats))
    _nvmlCheckReturn(ret)
    return c_fbcStats


def nvmlDeviceGetFBCSessions(handle):
    # first call to get the size
    c_session_count = c_uint(0)

    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCSessions")
    ret = fn(handle, byref(c_session_count), None)

    if (ret == NVML_SUCCESS):
        if (c_session_count.value != 0):
            # typical case
            session_array = c_nvmlFBCSession_t * c_session_count.value
            c_sessions = session_array()

            # make the call again
            ret = fn(handle, byref(c_session_count), c_sessions)
            _nvmlCheckReturn(ret)
            sessions = []
            for i in range(c_session_count.value):
                sessions.append(c_sessions[i])
            return sessions
        else:
            return []  # no active sessions
    else:
        # error case
        raise NVMLError(ret)


def nvmlVgpuInstanceGetEncoderStats(vgpuInstance):
    c_encoderCount = c_ulonglong(0)
    c_encodeFps = c_ulonglong(0)
    c_encoderLatency = c_ulonglong(0)
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderStats")
    ret = fn(vgpuInstance, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency))
    _nvmlCheckReturn(ret)
    return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value)


def nvmlVgpuInstanceGetEncoderSessions(vgpuInstance):
    # first call to get the size
    c_session_count = c_uint(0)

    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderSessions")
    ret = fn(vgpuInstance, byref(c_session_count), None)

    if (ret == NVML_SUCCESS):
        if (c_session_count.value != 0):
            # typical case
            session_array = c_nvmlEncoderSession_t * c_session_count.value
            c_sessions = session_array()

            # make the call again
            ret = fn(vgpuInstance, byref(c_session_count), c_sessions)
            _nvmlCheckReturn(ret)
            sessions = []
            for i in range(c_session_count.value):
                sessions.append(c_sessions[i])
            return sessions
        else:
            return []  # no active sessions
    else:
        # error case
        raise NVMLError(ret)


def nvmlVgpuInstanceGetFBCStats(vgpuInstance):
    c_fbcStats = c_nvmlFBCStats_t()
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCStats")
    ret = fn(vgpuInstance, byref(c_fbcStats))
    _nvmlCheckReturn(ret)
    return c_fbcStats


def nvmlVgpuInstanceGetFBCSessions(vgpuInstance):
    # first call to get the size
    c_session_count = c_uint(0)

    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCSessions")
    ret = fn(vgpuInstance, byref(c_session_count), None)

    if (ret == NVML_SUCCESS):
        if (c_session_count.value != 0):
            # typical case
            session_array = c_nvmlFBCSession_t * c_session_count.value
            c_sessions = session_array()

            # make the call again
            ret = fn(vgpuInstance, byref(c_session_count), c_sessions)
            _nvmlCheckReturn(ret)
            sessions = []
            for i in range(c_session_count.value):
                sessions.append(c_sessions[i])
            return sessions
        else:
            return []  # no active sessions
    else:
        # error case
        raise NVMLError(ret)


def nvmlDeviceGetProcessUtilization(handle, timeStamp):
    # first call to get the size
    c_count = c_uint(0)
    c_time_stamp = c_ulonglong(timeStamp)

    fn = _nvmlGetFunctionPointer("nvmlDeviceGetProcessUtilization")
    ret = fn(handle, None, byref(c_count), c_time_stamp)

    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        # typical case
        sampleArray = c_count.value * c_nvmlProcessUtilizationSample_t
        c_samples = sampleArray()

        # make the call again
        ret = fn(handle, c_samples, byref(c_count), c_time_stamp)
        _nvmlCheckReturn(ret)

        return c_samples[0:c_count.value]
    else:
        # error case
        raise NVMLError(ret)


def nvmlVgpuInstanceGetMetadata(vgpuInstance):
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMetadata")
    c_vgpuMetadata = c_nvmlVgpuMetadata_t()
    c_bufferSize = c_uint(0)
    # Make the first NVML API call to get the c_bufferSize value.
    # We have already allocated required buffer above.
    ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize))
    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize))
        _nvmlCheckReturn(ret)
    else:
        raise NVMLError(ret)
    return c_vgpuMetadata


def nvmlDeviceGetVgpuMetadata(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuMetadata")
    c_vgpuPgpuMetadata = c_nvmlVgpuPgpuMetadata_t()
    c_bufferSize = c_uint(0)
    # Make the first NVML API call to get the c_bufferSize value.
    # We have already allocated required buffer above.
    ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize))
    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize))
        _nvmlCheckReturn(ret)
    else:
        raise NVMLError(ret)
    return c_vgpuPgpuMetadata


def nvmlGetVgpuCompatibility(vgpuMetadata, pgpuMetadata):
    fn = _nvmlGetFunctionPointer("nvmlGetVgpuCompatibility")
    c_vgpuPgpuCompatibility = c_nvmlVgpuPgpuCompatibility_t()
    ret = fn(byref(vgpuMetadata), byref(pgpuMetadata), byref(c_vgpuPgpuCompatibility))
    _nvmlCheckReturn(ret)
    return c_vgpuPgpuCompatibility


def nvmlDeviceGetPgpuMetadataString(handle):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPgpuMetadataString")
    c_pgpuMetadata = create_string_buffer(NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE)
    c_bufferSize = c_uint(0)
    # Make the first NVML API call to get the c_bufferSize value.
    # We have already allocated required buffer above.
    ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize))
    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize))
        _nvmlCheckReturn(ret)
    else:
        raise NVMLError(ret)
    return (c_pgpuMetadata.value, c_bufferSize.value)


def nvmlSetVgpuVersion(vgpuVersion):
    fn = _nvmlGetFunctionPointer("nvmlSetVgpuVersion")
    ret = fn(byref(vgpuVersion))
    _nvmlCheckReturn(ret)
    return ret


def nvmlGetVgpuVersion(supported, current):
    fn = _nvmlGetFunctionPointer("nvmlGetVgpuVersion")
    ret = fn(byref(supported), byref(current))
    _nvmlCheckReturn(ret)
    return ret


def nvmlVgpuInstanceGetAccountingMode(vgpuInstance):
    c_mode = _nvmlEnableState_t()
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingMode")
    ret = fn(vgpuInstance, byref(c_mode))
    _nvmlCheckReturn(ret)
    return c_mode.value


def nvmlVgpuInstanceGetAccountingPids(vgpuInstance):
    c_pidCount = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingPids")
    ret = fn(vgpuInstance, byref(c_pidCount), None)
    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
        sampleArray = c_pidCount.value * c_uint
        c_pidArray = sampleArray()
        ret = fn(vgpuInstance, byref(c_pidCount), byref(c_pidArray))
        _nvmlCheckReturn(ret)
    else:
        raise NVMLError(ret)
    return (c_pidCount, c_pidArray)


def nvmlVgpuInstanceGetAccountingStats(vgpuInstance, pid):
    c_accountingStats = c_nvmlAccountingStats_t()
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingStats")
    ret = fn(vgpuInstance, pid, byref(c_accountingStats))
    _nvmlCheckReturn(ret)
    return c_accountingStats


def nvmlVgpuInstanceClearAccountingPids(vgpuInstance):
    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceClearAccountingPids")
    ret = fn(vgpuInstance)
    _nvmlCheckReturn(ret)
    return ret


def nvmlGetBlacklistDeviceCount():
    c_count = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlGetBlacklistDeviceCount")
    ret = fn(byref(c_count))
    _nvmlCheckReturn(ret)
    return c_count.value


def nvmlGetBlacklistDeviceInfoByIndex(index):
    c_index = c_uint(index)
    info = c_nvmlBlacklistDeviceInfo_t()
    fn = _nvmlGetFunctionPointer("nvmlGetBlacklistDeviceInfoByIndex")
    ret = fn(c_index, byref(info))
    _nvmlCheckReturn(ret)
    return info


def nvmlDeviceGetHostVgpuMode(handle):
    c_host_vgpu_mode = _nvmlHostVgpuMode_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHostVgpuMode")
    ret = fn(handle, byref(c_host_vgpu_mode))
    _nvmlCheckReturn(ret)
    return c_host_vgpu_mode.value


def nvmlDeviceSetMigMode(device, mode):
    c_activationStatus = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceSetMigMode")
    ret = fn(device, mode, byref(c_activationStatus))
    _nvmlCheckReturn(ret)
    return c_activationStatus.value


def nvmlDeviceGetMigMode(device):
    c_currentMode = c_uint()
    c_pendingMode = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMigMode")
    ret = fn(device, byref(c_currentMode), byref(c_pendingMode))
    _nvmlCheckReturn(ret)
    return [c_currentMode.value, c_pendingMode.value]


def nvmlDeviceGetGpuInstanceProfileInfo(device, profile):
    c_info = c_nvmlGpuInstanceProfileInfo_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceProfileInfo")
    ret = fn(device, profile, byref(c_info))
    _nvmlCheckReturn(ret)
    return c_info


def nvmlDeviceGetGpuInstanceRemainingCapacity(device, profileId):
    c_count = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceRemainingCapacity")
    ret = fn(device, profileId, byref(c_count))
    _nvmlCheckReturn(ret)
    return c_count.value


def nvmlDeviceGetGpuInstancePossiblePlacements(device, profileId, placementsRef, countRef):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstancePossiblePlacements")
    ret = fn(device, profileId, placementsRef, countRef)
    _nvmlCheckReturn(ret)
    return ret


def nvmlDeviceCreateGpuInstance(device, profileId):
    c_instance = c_nvmlGpuInstance_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceCreateGpuInstance")
    ret = fn(device, profileId, byref(c_instance))
    _nvmlCheckReturn(ret)
    return c_instance


def nvmlGpuInstanceDestroy(gpuInstance):
    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceDestroy")
    ret = fn(gpuInstance)
    _nvmlCheckReturn(ret)
    return ret


def nvmlDeviceGetGpuInstances(device, profileId, gpuInstancesRef, countRef):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstances")
    ret = fn(device, profileId, gpuInstancesRef, countRef)
    _nvmlCheckReturn(ret)
    return ret


def nvmlDeviceGetGpuInstanceById(device, gpuInstanceId):
    c_instance = c_nvmlGpuInstance_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceById")
    ret = fn(device, gpuInstanceId, byref(c_instance))
    _nvmlCheckReturn(ret)
    return c_instance


def nvmlGpuInstanceGetInfo(gpuInstance):
    c_info = c_nvmlGpuInstanceInfo_t()
    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetInfo")
    ret = fn(gpuInstance, byref(c_info))
    _nvmlCheckReturn(ret)
    return c_info


def nvmlGpuInstanceGetComputeInstanceProfileInfo(device, profile, engProfile):
    c_info = c_nvmlComputeInstanceProfileInfo_t()
    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfo")
    ret = fn(device, profile, engProfile, byref(c_info))
    _nvmlCheckReturn(ret)
    return c_info


def nvmlGpuInstanceGetComputeInstanceRemainingCapacity(gpuInstance, profileId):
    c_count = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceRemainingCapacity")
    ret = fn(gpuInstance, profileId, byref(c_count))
    _nvmlCheckReturn(ret)
    return c_count.value


def nvmlGpuInstanceCreateComputeInstance(gpuInstance, profileId):
    c_instance = c_nvmlComputeInstance_t()
    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceCreateComputeInstance")
    ret = fn(gpuInstance, profileId, byref(c_instance))
    _nvmlCheckReturn(ret)
    return c_instance


def nvmlComputeInstanceDestroy(computeInstance):
    fn = _nvmlGetFunctionPointer("nvmlComputeInstanceDestroy")
    ret = fn(computeInstance)
    _nvmlCheckReturn(ret)
    return ret


def nvmlGpuInstanceGetComputeInstances(gpuInstance, profileId, computeInstancesRef, countRef):
    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstances")
    ret = fn(gpuInstance, profileId, computeInstancesRef, countRef)
    _nvmlCheckReturn(ret)
    return ret


def nvmlGpuInstanceGetComputeInstanceById(gpuInstance, computeInstanceId):
    c_instance = c_nvmlComputeInstance_t()
    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceById")
    ret = fn(gpuInstance, computeInstanceId, byref(c_instance))
    _nvmlCheckReturn(ret)
    return c_instance


def nvmlComputeInstanceGetInfo(computeInstance):
    c_info = c_nvmlComputeInstanceInfo_t()
    fn = _nvmlGetFunctionPointer("nvmlComputeInstanceGetInfo")
    ret = fn(computeInstance, byref(c_info))
    _nvmlCheckReturn(ret)
    return c_info


def nvmlDeviceIsMigDeviceHandle(device):
    c_isMigDevice = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceIsMigDeviceHandle")
    ret = fn(device, byref(c_isMigDevice))
    _nvmlCheckReturn(ret)
    return c_isMigDevice


def nvmlDeviceGetGpuInstanceId(device):
    c_gpuInstanceId = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceId")
    ret = fn(device, byref(c_gpuInstanceId))
    _nvmlCheckReturn(ret)
    return c_gpuInstanceId.value


def nvmlDeviceGetComputeInstanceId(device):
    c_computeInstanceId = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeInstanceId")
    ret = fn(device, byref(c_computeInstanceId))
    _nvmlCheckReturn(ret)
    return c_computeInstanceId.value


def nvmlDeviceGetMaxMigDeviceCount(device):
    c_count = c_uint()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxMigDeviceCount")
    ret = fn(device, byref(c_count))
    _nvmlCheckReturn(ret)
    return c_count.value


def nvmlDeviceGetMigDeviceHandleByIndex(device, index):
    c_index = c_uint(index)
    migDevice = c_nvmlDevice_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMigDeviceHandleByIndex")
    ret = fn(device, c_index, byref(migDevice))
    _nvmlCheckReturn(ret)
    return migDevice


def nvmlDeviceGetDeviceHandleFromMigDeviceHandle(migDevice):
    device = c_nvmlDevice_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDeviceHandleFromMigDeviceHandle")
    ret = fn(migDevice, byref(device))
    _nvmlCheckReturn(ret)
    return device


def nvmlDeviceGetAttributes_v2(device):
    c_attrs = c_nvmlDeviceAttributes()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAttributes_v2")
    ret = fn(device, byref(c_attrs))
    _nvmlCheckReturn(ret)
    return c_attrs


def nvmlDeviceGetAttributes(device):
    return nvmlDeviceGetAttributes_v2(device)


def nvmlDeviceGetRemappedRows(device):
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRemappedRows")
    c_corr = c_uint()
    c_unc = c_uint()
    c_bpending = c_uint()
    c_bfailure = c_uint()
    ret = fn(device, byref(c_corr), byref(c_unc), byref(c_bpending), byref(c_bfailure))
    _nvmlCheckReturn(ret)
    return (c_corr.value, c_unc.value, c_bpending.value, c_bfailure.value)


def nvmlDeviceGetRowRemapperHistogram(device):
    c_vals = c_nvmlRowRemapperHistogramValues()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRowRemapperHistogram")
    ret = fn(device, byref(c_vals))
    _nvmlCheckReturn(ret)
    return c_vals


def nvmlDeviceGetArchitecture(device):
    arch = _nvmlDeviceArchitecture_t()
    fn = _nvmlGetFunctionPointer("nvmlDeviceGetArchitecture")
    ret = fn(device, byref(arch))
    _nvmlCheckReturn(ret)
    return arch.value


# clearml/utilities/locks/__init__.py

from .utils import RLock, Lock

__all__ = ["RLock", "Lock"]


# clearml/utilities/locks/constants.py

"""
Locking constants

Lock types:

- `LOCK_EX` exclusive lock
- `LOCK_SH` shared lock

Lock flags:

- `LOCK_NB` non-blocking

Manually unlock, only needed internally

- `LOCK_UN` unlock
"""
import os

# The actual tests will execute the code anyhow so the following code can
# safely be ignored from the coverage tests
if os.name == 'nt':  # pragma: no cover
    import msvcrt

    LOCK_EX = 0x1  #: exclusive lock
    LOCK_SH = 0x2  #: shared lock
    LOCK_NB = 0x4  #: non-blocking
    LOCK_UN = msvcrt.LK_UNLCK  #: unlock

    LOCKFILE_FAIL_IMMEDIATELY = 1
    LOCKFILE_EXCLUSIVE_LOCK = 2

elif os.name == 'posix':  # pragma: no cover
    import fcntl

    LOCK_EX = fcntl.LOCK_EX  #: exclusive lock
    LOCK_SH = fcntl.LOCK_SH  #: shared lock
    LOCK_NB = fcntl.LOCK_NB  #: non-blocking
    LOCK_UN = fcntl.LOCK_UN  #: unlock

else:  # pragma: no cover
    raise RuntimeError('PortaLocker only defined for nt and posix platforms')


# clearml/utilities/locks/exceptions.py

class BaseLockException(Exception):
    # Error codes:
    LOCK_FAILED = 1

    def __init__(self, *args, **kwargs):
        self.fh = kwargs.pop('fh', None)
        Exception.__init__(self, *args, **kwargs)


class LockException(BaseLockException):
    pass


class AlreadyLocked(BaseLockException):
    pass


class FileToLarge(BaseLockException):
    pass


# clearml/utilities/locks/portalocker.py

import os
import sys
from . import exceptions
from . import constants


if os.name == 'nt':  # pragma: no cover
    import msvcrt

    if sys.version_info.major == 2:
        lock_length = -1
    else:
        lock_length = int(2**31 - 1)

    def lock(file_, flags):
        if flags & constants.LOCK_SH:
            import win32file
            import pywintypes
            import winerror
            __overlapped = pywintypes.OVERLAPPED()
            if sys.version_info.major == 2:
                if flags & constants.LOCK_NB:
                    mode = constants.LOCKFILE_FAIL_IMMEDIATELY
                else:
                    mode = 0

            else:
                if flags & constants.LOCK_NB:
                    mode = msvcrt.LK_NBRLCK
                else:
                    mode = msvcrt.LK_RLCK

            # is there any reason not to reuse the following structure?
            hfile = win32file._get_osfhandle(file_.fileno())
            try:
                win32file.LockFileEx(hfile, mode, 0, -0x10000, __overlapped)
            except pywintypes.error as exc_value:
                # error: (33, 'LockFileEx', 'The process cannot access the file
                # because another process has locked a portion of the file.')
                if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION:
                    raise exceptions.LockException(
                        exceptions.LockException.LOCK_FAILED,
                        exc_value.strerror,
                        fh=file_)
                else:
                    # Q:  Are there exceptions/codes we should be dealing with
                    # here?
                    raise
        else:
            mode = constants.LOCKFILE_EXCLUSIVE_LOCK
            if flags & constants.LOCK_NB:
                mode |= constants.LOCKFILE_FAIL_IMMEDIATELY

            if flags & constants.LOCK_NB:
                mode = msvcrt.LK_NBLCK
            else:
                mode = msvcrt.LK_LOCK

            # windows locks byte ranges, so make sure to lock from file start
            try:
                savepos = file_.tell()
                if savepos:
                    # [ ] test exclusive lock fails on seek here
                    # [ ] test if shared lock passes this point
                    file_.seek(0)
                    # [x] check if 0 param locks entire file (not documented in
                    #     Python)
                    # [x] fails with "IOError: [Errno 13] Permission denied",
                    #     but -1 seems to do the trick

                try:
                    msvcrt.locking(file_.fileno(), mode, lock_length)
                except IOError as exc_value:
                    # [ ] be more specific here
                    raise exceptions.LockException(
                        exceptions.LockException.LOCK_FAILED,
                        exc_value.strerror,
                        fh=file_)
                finally:
                    if savepos:
                        file_.seek(savepos)
            except IOError as exc_value:
                raise exceptions.LockException(
                    exceptions.LockException.LOCK_FAILED, exc_value.strerror,
                    fh=file_)

    def unlock(file_):
        try:
            savepos = file_.tell()
            if savepos:
                file_.seek(0)

            try:
                msvcrt.locking(file_.fileno(), constants.LOCK_UN, lock_length)
            except IOError as exc_value:
                if exc_value.strerror == 'Permission denied':
                    import pywintypes
                    import win32file
                    import winerror
                    __overlapped = pywintypes.OVERLAPPED()
                    hfile = win32file._get_osfhandle(file_.fileno())
                    try:
                        win32file.UnlockFileEx(
                            hfile, 0, -0x10000, __overlapped)
                    except pywintypes.error as exc_value:
                        if exc_value.winerror == winerror.ERROR_NOT_LOCKED:
                            # error: (158, 'UnlockFileEx',
                            #         'The segment is already unlocked.')
                            # To match the 'posix' implementation, silently
                            # ignore this error
                            pass
                        else:
                            # Q:  Are there exceptions/codes we should be
                            # dealing with here?
                            raise
                else:
                    raise exceptions.LockException(
                        exceptions.LockException.LOCK_FAILED,
                        exc_value.strerror,
                        fh=file_)
            finally:
                if savepos:
                    file_.seek(savepos)
        except IOError as exc_value:
            raise exceptions.LockException(
                exceptions.LockException.LOCK_FAILED, exc_value.strerror,
                fh=file_)

elif os.name == 'posix':  # pragma: no cover
    import fcntl

    def lock(file_, flags):
        locking_exceptions = IOError,
        try:  # pragma: no cover
            locking_exceptions += BlockingIOError,
        except NameError:  # pragma: no cover
            pass

        try:
            fcntl.flock(file_.fileno(), flags)
        except locking_exceptions as exc_value:
            # The exception code varies on different systems so we'll catch
            # every IO error
            raise exceptions.LockException(exc_value, fh=file_)

    def unlock(file_):
        fcntl.flock(file_.fileno(), constants.LOCK_UN)

else:  # pragma: no cover
    raise RuntimeError('PortaLocker only defined for nt and posix platforms')


# clearml/utilities/locks/utils.py

import os
import time
import atexit
import tempfile
import contextlib
from multiprocessing import RLock as ProcessRLock
from . import exceptions
from . import constants
from . import portalocker

current_time = getattr(time, "monotonic", time.time)

DEFAULT_TIMEOUT = 10 ** 8
DEFAULT_CHECK_INTERVAL = 0.25
LOCK_METHOD = constants.LOCK_EX | constants.LOCK_NB

__all__ = [
    'Lock',
    'RLock',
    'open_atomic',
]


@contextlib.contextmanager
def open_atomic(filename, binary=True):
    '''Open a file for atomic writing. Instead of locking this method allows
    you to write the entire file and move it to the actual location. Note that
    this makes the assumption that a rename is atomic on your platform which
    is generally the case but not a guarantee.

    http://docs.python.org/library/os.html#os.rename

    >>> filename = 'test_file.txt'
    >>> if os.path.exists(filename):
    ...     os.remove(filename)

    >>> with open_atomic(filename) as fh:
    ...     written = fh.write(b'test')
    >>> assert os.path.exists(filename)
    >>> os.remove(filename)

    '''
    assert not os.path.exists(filename), '%r exists' % filename
    path, name = os.path.split(filename)

    # Create the parent directory if it doesn't exist
    if path and not os.path.isdir(path):  # pragma: no cover
        os.makedirs(path)

    temp_fh = tempfile.NamedTemporaryFile(
        mode=binary and 'wb' or 'w',
        dir=path,
        delete=False,
    )
    yield temp_fh
    temp_fh.flush()
    os.fsync(temp_fh.fileno())
    temp_fh.close()
    try:
        os.rename(temp_fh.name, filename)
    finally:
        try:
            os.remove(temp_fh.name)
        except Exception:
            pass


class Lock(object):

    def __init__(
            self, filename, mode='a', timeout=DEFAULT_TIMEOUT,
            check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False,
            flags=LOCK_METHOD, **file_open_kwargs):
        '''Lock manager with build-in timeout

        filename -- filename
        mode -- the open mode, 'a' or 'ab' should be used for writing
        truncate -- use truncate to emulate 'w' mode, None is disabled, 0 is
            truncate to 0 bytes
        timeout -- timeout when trying to acquire a lock
        check_interval -- check interval while waiting
        fail_when_locked -- after the initial lock failed, return an error
            or lock the file
        **file_open_kwargs -- The kwargs for the `open(...)` call

        fail_when_locked is useful when multiple threads/processes can race
        when creating a file. If set to true than the system will wait till
        the lock was acquired and then return an AlreadyLocked exception.

        Note that the file is opened first and locked later. So using 'w' as
        mode will result in truncate _BEFORE_ the lock is checked.
        '''

        if 'w' in mode:
            truncate = True
            mode = mode.replace('w', 'a')
        else:
            truncate = False

        self.fh = None
        self.filename = filename
        self.mode = mode
        self.truncate = truncate
        self.timeout = timeout
        self.check_interval = check_interval
        self.fail_when_locked = fail_when_locked
        self.flags = flags
        self.file_open_kwargs = file_open_kwargs

    def acquire(
            self, timeout=None, check_interval=None, fail_when_locked=None):
        '''Acquire the locked filehandle'''
        if timeout is None:
            timeout = self.timeout
        if timeout is None:
            timeout = 0

        if check_interval is None:
            check_interval = self.check_interval

        if fail_when_locked is None:
            fail_when_locked = self.fail_when_locked

        # If we already have a filehandle, return it
        fh = self.fh
        if fh:
            return fh

        # Get a new filehandler
        fh = self._get_fh()
        try:
            # Try to lock
            fh = self._get_lock(fh)
        except exceptions.LockException as exception:
            # Try till the timeout has passed
            timeoutend = current_time() + timeout
            while timeoutend > current_time():
                # Wait a bit
                time.sleep(check_interval)

                # Try again
                try:

                    # We already tried to the get the lock
                    # If fail_when_locked is true, then stop trying
                    if fail_when_locked:
                        raise exceptions.AlreadyLocked(exception)

                    else:  # pragma: no cover
                        # We've got the lock
                        fh = self._get_lock(fh)
                        break

                except exceptions.LockException:
                    pass

            else:
                # We got a timeout... reraising
                raise exceptions.LockException(exception)

        # Prepare the filehandle (truncate if needed)
        fh = self._prepare_fh(fh)

        self.fh = fh
        return fh

    def release(self):
        '''Releases the currently locked file handle'''
        if self.fh:
            # noinspection PyBroadException
            try:
                portalocker.unlock(self.fh)
            except Exception:
                pass
            # noinspection PyBroadException
            try:
                self.fh.close()
            except Exception:
                pass
            self.fh = None

    def delete_lock_file(self):
        # type: () -> bool
        """
        Remove the local file used for locking (fail if file is locked)

        :return: True is successful
        """
        if self.fh:
            return False
        # noinspection PyBroadException
        try:
            os.unlink(path=self.filename)
        except BaseException:
            return False
        return True

    def _get_fh(self):
        '''Get a new filehandle'''
        # Create the parent directory if it doesn't exist
        path, name = os.path.split(self.filename)
        if path and not os.path.isdir(path):  # pragma: no cover
            os.makedirs(path, exist_ok=True)

        return open(self.filename, self.mode, **self.file_open_kwargs)

    def _get_lock(self, fh):
        '''
        Try to lock the given filehandle

        returns LockException if it fails'''
        portalocker.lock(fh, self.flags)
        return fh

    def _prepare_fh(self, fh):
        '''
        Prepare the filehandle for usage

        If truncate is a number, the file will be truncated to that amount of
        bytes
        '''
        if self.truncate:
            fh.seek(0)
            fh.truncate(0)

        return fh

    def __enter__(self):
        return self.acquire()

    def __exit__(self, type_, value, tb):
        self.release()

    def __delete__(self, instance):  # pragma: no cover
        instance.release()


class RLock(Lock):
    """
    A reentrant lock, functions in a similar way to threading.RLock in that it
    can be acquired multiple times.  When the corresponding number of release()
    calls are made the lock will finally release the underlying file lock.
    """

    def __init__(
            self, filename, mode='a', timeout=DEFAULT_TIMEOUT,
            check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False,
            flags=LOCK_METHOD):
        super(RLock, self).__init__(filename, mode, timeout, check_interval,
                                    fail_when_locked, flags)
        self._acquire_count = 0
        self._lock = ProcessRLock()
        self._pid = os.getpid()

    def acquire(self, timeout=None, check_interval=None, fail_when_locked=None):
        if self._lock:
            # cleanup bad python behaviour when forking while lock is acquired
            # see Issue https://github.com/allegroai/clearml-agent/issues/73
            # and https://bugs.python.org/issue6721
            if self._pid != os.getpid():
                # noinspection PyBroadException
                try:
                    if self._lock._semlock._count():  # noqa
                        # this should never happen unless python forgot calling _after_fork
                        self._lock._semlock._after_fork()  # noqa
                except BaseException:
                    pass

            if not self._lock.acquire(block=timeout != 0, timeout=timeout):
                # We got a timeout... reraising
                raise exceptions.LockException()

        # check if we need to recreate the file lock on another subprocess
        if self._pid != os.getpid():
            self._pid = os.getpid()
            self._acquire_count = 0
            if self.fh:
                # noinspection PyBroadException
                try:
                    portalocker.unlock(self.fh)
                    self.fh.close()
                except Exception:
                    pass
            self.fh = None

        if self._acquire_count >= 1:
            fh = self.fh
        else:
            fh = super(RLock, self).acquire(timeout, check_interval,
                                            fail_when_locked)

        self._acquire_count += 1
        return fh

    def release(self):
        if self._acquire_count == 0:
            raise exceptions.LockException(
                "Cannot release more times than acquired")

        if self._acquire_count == 1:
            super(RLock, self).release()

        self._acquire_count -= 1
        if self._lock:
            self._lock.release()

    def __del__(self):
        self._lock = None
        # try to remove the file when we are done
        if not os.path.isfile(self.filename):
            return
        try:
            self.acquire(timeout=0)
            try:
                os.unlink(self.filename)
                removed = True
            except Exception:
                removed = False
            self.release()
            if not removed:
                try:
                    os.unlink(self.filename)
                except Exception:
                    pass
        except Exception:
            pass


class TemporaryFileLock(Lock):

    def __init__(self, filename='.lock', timeout=DEFAULT_TIMEOUT,
                 check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=True,
                 flags=LOCK_METHOD):

        Lock.__init__(self, filename=filename, mode='w', timeout=timeout,
                      check_interval=check_interval,
                      fail_when_locked=fail_when_locked, flags=flags)
        atexit.register(self.release)

    def release(self):
        Lock.release(self)
        if os.path.isfile(self.filename):  # pragma: no branch
            os.unlink(self.filename)


# clearml/utilities/lowlevel/astor_unparse.py

from __future__ import print_function, unicode_literals

import ast
import sys
import six
from six import StringIO

# Large float and imaginary literals get turned into infinities in the AST.
# We unparse those infinities to INFSTR.
INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)


def interleave(inter, f, seq):
    """Call f on each item in seq, calling inter() in between.
    """
    seq = iter(seq)
    try:
        f(next(seq))
    except StopIteration:
        pass
    else:
        for x in seq:
            inter()
            f(x)


class Unparser:
    """Methods in this class recursively traverse an AST and
    output source code for the abstract syntax; original formatting
    is disregarded. """

    def __init__(self, tree, file=sys.stdout):
        """Unparser(tree, file=sys.stdout) -> None.
         Print the source for tree to file."""
        self.f = file
        self.future_imports = []
        self._indent = 0
        self.dispatch(tree)
        print("", file=self.f)
        self.f.flush()

    def fill(self, text=""):
        """Indent a piece of text, according to the current indentation level"""
        self.f.write("\n" + "    " * self._indent + text)

    def write(self, text):
        """Append a piece of text to the current line."""
        self.f.write(six.text_type(text))

    def enter(self):
        """Print ':', and increase the indentation."""
        self.write(":")
        self._indent += 1

    def leave(self):
        """Decrease the indentation level."""
        self._indent -= 1

    def dispatch(self, tree):
        """
        Dispatcher function, dispatching tree type T to method _T.
        """
        if isinstance(tree, list):
            for t in tree:
                self.dispatch(t)
            return
        meth = getattr(self, "_" + tree.__class__.__name__)
        meth(tree)

    """
    ############### Unparsing methods ######################
    # There should be one method per concrete grammar type #
    # Constructors should be grouped by sum type. Ideally, #
    # this would follow the order in the grammar, but      #
    # currently doesn't.                                   #
    ########################################################
    """

    def _Module(self, tree):
        for stmt in tree.body:
            self.dispatch(stmt)

    def _Interactive(self, tree):
        for stmt in tree.body:
            self.dispatch(stmt)

    def _Expression(self, tree):
        self.dispatch(tree.body)

    # stmt
    def _Expr(self, tree):
        self.fill()
        self.dispatch(tree.value)

    def _NamedExpr(self, tree):
        self.write("(")
        self.dispatch(tree.target)
        self.write(" := ")
        self.dispatch(tree.value)
        self.write(")")

    def _Import(self, t):
        self.fill("import ")
        interleave(lambda: self.write(", "), self.dispatch, t.names)

    def _ImportFrom(self, t):
        # A from __future__ import may affect unparsing, so record it.
        if t.module and t.module == '__future__':
            self.future_imports.extend(n.name for n in t.names)

        self.fill("from ")
        self.write("." * t.level)
        if t.module:
            self.write(t.module)
        self.write(" import ")
        interleave(lambda: self.write(", "), self.dispatch, t.names)

    def _Assign(self, t):
        self.fill()
        for target in t.targets:
            self.dispatch(target)
            self.write(" = ")
        self.dispatch(t.value)

    def _AugAssign(self, t):
        self.fill()
        self.dispatch(t.target)
        self.write(" " + self.binop[t.op.__class__.__name__] + "= ")
        self.dispatch(t.value)

    def _AnnAssign(self, t):
        self.fill()
        if not t.simple and isinstance(t.target, ast.Name):
            self.write('(')
        self.dispatch(t.target)
        if not t.simple and isinstance(t.target, ast.Name):
            self.write(')')
        self.write(": ")
        self.dispatch(t.annotation)
        if t.value:
            self.write(" = ")
            self.dispatch(t.value)

    def _Return(self, t):
        self.fill("return")
        if t.value:
            self.write(" ")
            self.dispatch(t.value)

    def _Pass(self, t):
        self.fill("pass")

    def _Break(self, t):
        self.fill("break")

    def _Continue(self, t):
        self.fill("continue")

    def _Delete(self, t):
        self.fill("del ")
        interleave(lambda: self.write(", "), self.dispatch, t.targets)

    def _Assert(self, t):
        self.fill("assert ")
        self.dispatch(t.test)
        if t.msg:
            self.write(", ")
            self.dispatch(t.msg)

    def _Exec(self, t):
        self.fill("exec ")
        self.dispatch(t.body)
        if t.globals:
            self.write(" in ")
            self.dispatch(t.globals)
        if t.locals:
            self.write(", ")
            self.dispatch(t.locals)

    def _Print(self, t):
        self.fill("print ")
        do_comma = False
        if t.dest:
            self.write(">>")
            self.dispatch(t.dest)
            do_comma = True
        for e in t.values:
            if do_comma:
                self.write(", ")
            else:
                do_comma = True
            self.dispatch(e)
        if not t.nl:
            self.write(",")

    def _Global(self, t):
        self.fill("global ")
        interleave(lambda: self.write(", "), self.write, t.names)

    def _Nonlocal(self, t):
        self.fill("nonlocal ")
        interleave(lambda: self.write(", "), self.write, t.names)

    def _Await(self, t):
        self.write("(")
        self.write("await")
        if t.value:
            self.write(" ")
            self.dispatch(t.value)
        self.write(")")

    def _Yield(self, t):
        self.write("(")
        self.write("yield")
        if t.value:
            self.write(" ")
            self.dispatch(t.value)
        self.write(")")

    def _YieldFrom(self, t):
        self.write("(")
        self.write("yield from")
        if t.value:
            self.write(" ")
            self.dispatch(t.value)
        self.write(")")

    def _Raise(self, t):
        self.fill("raise")
        if six.PY3:
            if not t.exc:
                assert not t.cause
                return
            self.write(" ")
            self.dispatch(t.exc)
            if t.cause:
                self.write(" from ")
                self.dispatch(t.cause)
        else:
            self.write(" ")
            if t.type:
                self.dispatch(t.type)
            if t.inst:
                self.write(", ")
                self.dispatch(t.inst)
            if t.tback:
                self.write(", ")
                self.dispatch(t.tback)

    def _Try(self, t):
        self.fill("try")
        self.enter()
        self.dispatch(t.body)
        self.leave()
        for ex in t.handlers:
            self.dispatch(ex)
        if t.orelse:
            self.fill("else")
            self.enter()
            self.dispatch(t.orelse)
            self.leave()
        if t.finalbody:
            self.fill("finally")
            self.enter()
            self.dispatch(t.finalbody)
            self.leave()

    def _TryExcept(self, t):
        self.fill("try")
        self.enter()
        self.dispatch(t.body)
        self.leave()

        for ex in t.handlers:
            self.dispatch(ex)
        if t.orelse:
            self.fill("else")
            self.enter()
            self.dispatch(t.orelse)
            self.leave()

    def _TryFinally(self, t):
        if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept):
            # try-except-finally
            self.dispatch(t.body)
        else:
            self.fill("try")
            self.enter()
            self.dispatch(t.body)
            self.leave()

        self.fill("finally")
        self.enter()
        self.dispatch(t.finalbody)
        self.leave()

    def _ExceptHandler(self, t):
        self.fill("except")
        if t.type:
            self.write(" ")
            self.dispatch(t.type)
        if t.name:
            self.write(" as ")
            if six.PY3:
                self.write(t.name)
            else:
                self.dispatch(t.name)
        self.enter()
        self.dispatch(t.body)
        self.leave()

    def _ClassDef(self, t):
        self.write("\n")
        for deco in t.decorator_list:
            self.fill("@")
            self.dispatch(deco)
        self.fill("class " + t.name)
        if six.PY3:
            self.write("(")
            comma = False
            for e in t.bases:
                if comma:
                    self.write(", ")
                else:
                    comma = True
                self.dispatch(e)
            for e in t.keywords:
                if comma:
                    self.write(", ")
                else:
                    comma = True
                self.dispatch(e)
            if sys.version_info[:2] < (3, 5):
                if t.starargs:
                    if comma:
                        self.write(", ")
                    else:
                        comma = True
                    self.write("*")
                    self.dispatch(t.starargs)
                if t.kwargs:
                    if comma:
                        self.write(", ")
                    else:
                        comma = True
                    self.write("**")
                    self.dispatch(t.kwargs)
            self.write(")")
        elif t.bases:
            self.write("(")
            for a in t.bases:
                self.dispatch(a)
                self.write(", ")
            self.write(")")
        self.enter()
        self.dispatch(t.body)
        self.leave()

    def _FunctionDef(self, t):
        self.__FunctionDef_helper(t, "def")

    def _AsyncFunctionDef(self, t):
        self.__FunctionDef_helper(t, "async def")

    def __FunctionDef_helper(self, t, fill_suffix):
        self.write("\n")
        for deco in t.decorator_list:
            self.fill("@")
            self.dispatch(deco)
        def_str = fill_suffix + " " + t.name + "("
        self.fill(def_str)
        self.dispatch(t.args)
        self.write(")")
        if getattr(t, "returns", False):
            self.write(" -> ")
            self.dispatch(t.returns)
        self.enter()
        self.dispatch(t.body)
        self.leave()

    def _For(self, t):
        self.__For_helper("for ", t)

    def _AsyncFor(self, t):
        self.__For_helper("async for ", t)

    def __For_helper(self, fill, t):
        self.fill(fill)
        self.dispatch(t.target)
        self.write(" in ")
        self.dispatch(t.iter)
        self.enter()
        self.dispatch(t.body)
        self.leave()
        if t.orelse:
            self.fill("else")
            self.enter()
            self.dispatch(t.orelse)
            self.leave()

    def _If(self, t):
        self.fill("if ")
        self.dispatch(t.test)
        self.enter()
        self.dispatch(t.body)
        self.leave()
        # collapse nested ifs into equivalent elifs.
        while (t.orelse and len(t.orelse) == 1 and
               isinstance(t.orelse[0], ast.If)):
            t = t.orelse[0]
            self.fill("elif ")
            self.dispatch(t.test)
            self.enter()
            self.dispatch(t.body)
            self.leave()
        # final else
        if t.orelse:
            self.fill("else")
            self.enter()
            self.dispatch(t.orelse)
            self.leave()

    def _While(self, t):
        self.fill("while ")
        self.dispatch(t.test)
        self.enter()
        self.dispatch(t.body)
        self.leave()
        if t.orelse:
            self.fill("else")
            self.enter()
            self.dispatch(t.orelse)
            self.leave()

    def _generic_With(self, t, async_=False):
        self.fill("async with " if async_ else "with ")
        if hasattr(t, 'items'):
            interleave(lambda: self.write(", "), self.dispatch, t.items)
        else:
            self.dispatch(t.context_expr)
            if t.optional_vars:
                self.write(" as ")
                self.dispatch(t.optional_vars)
        self.enter()
        self.dispatch(t.body)
        self.leave()

    def _With(self, t):
        self._generic_With(t)

    def _AsyncWith(self, t):
        self._generic_With(t, async_=True)

    # expr
    def _Bytes(self, t):
        self.write(repr(t.s))

    def _Str(self, tree):
        if six.PY3:
            self.write(repr(tree.s))
        else:
            # if from __future__ import unicode_literals is in effect,
            # then we want to output string literals using a 'b' prefix
            # and unicode literals with no prefix.
            if "unicode_literals" not in self.future_imports:
                self.write(repr(tree.s))
            elif isinstance(tree.s, str):
                self.write("b" + repr(tree.s))
            elif isinstance(tree.s, unicode):  # noqa
                self.write(repr(tree.s).lstrip("u"))
            else:
                assert False, "shouldn't get here"

    def _JoinedStr(self, t):
        # JoinedStr(expr* values)
        self.write("f")
        string = StringIO()
        self._fstring_JoinedStr(t, string.write)
        # Deviation from `unparse.py`: Try to find an unused quote.
        # This change is made to handle _very_ complex f-strings.
        v = string.getvalue()
        if '\n' in v or '\r' in v:
            quote_types = ["'''", '"""']
        else:
            quote_types = ["'", '"', '"""', "'''"]
        for quote_type in quote_types:
            if quote_type not in v:
                v = "{quote_type}{v}{quote_type}".format(quote_type=quote_type, v=v)
                break
        else:
            v = repr(v)
        self.write(v)

    def _FormattedValue(self, t):
        # FormattedValue(expr value, int? conversion, expr? format_spec)
        self.write("f")
        string = StringIO()
        self._fstring_JoinedStr(t, string.write)
        self.write(repr(string.getvalue()))

    def _fstring_JoinedStr(self, t, write):
        for value in t.values:
            meth = getattr(self, "_fstring_" + type(value).__name__)
            meth(value, write)

    def _fstring_Str(self, t, write):
        value = t.s.replace("{", "{{").replace("}", "}}")
        write(value)

    def _fstring_Constant(self, t, write):
        assert isinstance(t.value, str)
        value = t.value.replace("{", "{{").replace("}", "}}")
        write(value)

    def _fstring_FormattedValue(self, t, write):
        write("{")
        expr = StringIO()
        Unparser(t.value, expr)
        expr = expr.getvalue().rstrip("\n")
        if expr.startswith("{"):
            write(" ")  # Separate pair of opening brackets as "{ {"
        write(expr)
        if t.conversion != -1:
            conversion = chr(t.conversion)
            assert conversion in "sra"
            write("!{conversion}".format(conversion=conversion))
        if t.format_spec:
            write(":")
            meth = getattr(self, "_fstring_" + type(t.format_spec).__name__)
            meth(t.format_spec, write)
        write("}")

    def _Name(self, t):
        self.write(t.id)

    def _NameConstant(self, t):
        self.write(repr(t.value))

    def _Repr(self, t):
        self.write("`")
        self.dispatch(t.value)
        self.write("`")

    def _write_constant(self, value):
        if isinstance(value, (float, complex)):
            # Substitute overflowing decimal literal for AST infinities.
            self.write(repr(value).replace("inf", INFSTR))
        else:
            self.write(repr(value))

    def _Constant(self, t):
        value = t.value
        if isinstance(value, tuple):
            self.write("(")
            if len(value) == 1:
                self._write_constant(value[0])
                self.write(",")
            else:
                interleave(lambda: self.write(", "), self._write_constant, value)
            self.write(")")
        elif value is Ellipsis:  # instead of `...` for Py2 compatibility
            self.write("...")
        else:
            if t.kind == "u":
                self.write("u")
            self._write_constant(t.value)

    def _Num(self, t):
        repr_n = repr(t.n)
        if six.PY3:
            self.write(repr_n.replace("inf", INFSTR))
        else:
            # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2.
            if repr_n.startswith("-"):
                self.write("(")
            if "inf" in repr_n and repr_n.endswith("*j"):
                repr_n = repr_n.replace("*j", "j")
            # Substitute overflowing decimal literal for AST infinities.
            self.write(repr_n.replace("inf", INFSTR))
            if repr_n.startswith("-"):
                self.write(")")

    def _List(self, t):
        self.write("[")
        interleave(lambda: self.write(", "), self.dispatch, t.elts)
        self.write("]")

    def _ListComp(self, t):
        self.write("[")
        self.dispatch(t.elt)
        for gen in t.generators:
            self.dispatch(gen)
        self.write("]")

    def _GeneratorExp(self, t):
        self.write("(")
        self.dispatch(t.elt)
        for gen in t.generators:
            self.dispatch(gen)
        self.write(")")

    def _SetComp(self, t):
        self.write("{")
        self.dispatch(t.elt)
        for gen in t.generators:
            self.dispatch(gen)
        self.write("}")

    def _DictComp(self, t):
        self.write("{")
        self.dispatch(t.key)
        self.write(": ")
        self.dispatch(t.value)
        for gen in t.generators:
            self.dispatch(gen)
        self.write("}")

    def _comprehension(self, t):
        if getattr(t, 'is_async', False):
            self.write(" async for ")
        else:
            self.write(" for ")
        self.dispatch(t.target)
        self.write(" in ")
        self.dispatch(t.iter)
        for if_clause in t.ifs:
            self.write(" if ")
            self.dispatch(if_clause)

    def _IfExp(self, t):
        self.write("(")
        self.dispatch(t.body)
        self.write(" if ")
        self.dispatch(t.test)
        self.write(" else ")
        self.dispatch(t.orelse)
        self.write(")")

    def _Set(self, t):
        assert (t.elts)  # should be at least one element
        self.write("{")
        interleave(lambda: self.write(", "), self.dispatch, t.elts)
        self.write("}")

    def _Dict(self, t):
        self.write("{")

        def write_key_value_pair(k, v):
            self.dispatch(k)
            self.write(": ")
            self.dispatch(v)

        def write_item(item):
            k, v = item
            if k is None:
                # for dictionary unpacking operator in dicts {**{'y': 2}}
                # see PEP 448 for details
                self.write("**")
                self.dispatch(v)
            else:
                write_key_value_pair(k, v)

        interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values))
        self.write("}")

    def _Tuple(self, t):
        self.write("(")
        if len(t.elts) == 1:
            elt = t.elts[0]
            self.dispatch(elt)
            self.write(",")
        else:
            interleave(lambda: self.write(", "), self.dispatch, t.elts)
        self.write(")")

    unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"}

    def _UnaryOp(self, t):
        self.write("(")
        self.write(self.unop[t.op.__class__.__name__])
        self.write(" ")
        if six.PY2 and isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num):
            # If we're applying unary minus to a number, parenthesize the number.
            # This is necessary: -2147483648 is different from -(2147483648) on
            # a 32-bit machine (the first is an int, the second a long), and
            # -7j is different from -(7j).  (The first has real part 0.0, the second
            # has real part -0.0.)
            self.write("(")
            self.dispatch(t.operand)
            self.write(")")
        else:
            self.dispatch(t.operand)
        self.write(")")

    binop = {"Add": "+", "Sub": "-", "Mult": "*", "MatMult": "@", "Div": "/", "Mod": "%",
             "LShift": "<<", "RShift": ">>", "BitOr": "|", "BitXor": "^", "BitAnd": "&",
             "FloorDiv": "//", "Pow": "**"}

    def _BinOp(self, t):
        self.write("(")
        self.dispatch(t.left)
        self.write(" " + self.binop[t.op.__class__.__name__] + " ")
        self.dispatch(t.right)
        self.write(")")

    cmpops = {"Eq": "==", "NotEq": "!=", "Lt": "<", "LtE": "<=", "Gt": ">", "GtE": ">=",
              "Is": "is", "IsNot": "is not", "In": "in", "NotIn": "not in"}

    def _Compare(self, t):
        self.write("(")
        self.dispatch(t.left)
        for o, e in zip(t.ops, t.comparators):
            self.write(" " + self.cmpops[o.__class__.__name__] + " ")
            self.dispatch(e)
        self.write(")")

    boolops = {ast.And: 'and', ast.Or: 'or'}

    def _BoolOp(self, t):
        self.write("(")
        s = " %s " % self.boolops[t.op.__class__]
        interleave(lambda: self.write(s), self.dispatch, t.values)
        self.write(")")

    def _Attribute(self, t):
        self.dispatch(t.value)
        # Special case: 3.__abs__() is a syntax error, so if t.value
        # is an integer literal then we need to either parenthesize
        # it or add an extra space to get 3 .__abs__().
        if isinstance(t.value, getattr(ast, 'Constant', getattr(ast, 'Num', None))) and isinstance(t.value.n, int):
            self.write(" ")
        self.write(".")
        self.write(t.attr)

    def _Call(self, t):
        self.dispatch(t.func)
        self.write("(")
        comma = False
        for e in t.args:
            if comma:
                self.write(", ")
            else:
                comma = True
            self.dispatch(e)
        for e in t.keywords:
            if comma:
                self.write(", ")
            else:
                comma = True
            self.dispatch(e)
        if sys.version_info[:2] < (3, 5):
            if t.starargs:
                if comma:
                    self.write(", ")
                else:
                    comma = True
                self.write("*")
                self.dispatch(t.starargs)
            if t.kwargs:
                if comma:
                    self.write(", ")
                else:
                    comma = True
                self.write("**")
                self.dispatch(t.kwargs)
        self.write(")")

    def _Subscript(self, t):
        self.dispatch(t.value)
        self.write("[")
        self.dispatch(t.slice)
        self.write("]")

    def _Starred(self, t):
        self.write("*")
        self.dispatch(t.value)

    # slice
    def _Ellipsis(self, t):
        self.write("...")

    def _Index(self, t):
        self.dispatch(t.value)

    def _Slice(self, t):
        if t.lower:
            self.dispatch(t.lower)
        self.write(":")
        if t.upper:
            self.dispatch(t.upper)
        if t.step:
            self.write(":")
            self.dispatch(t.step)

    def _ExtSlice(self, t):
        interleave(lambda: self.write(', '), self.dispatch, t.dims)

    # argument
    def _arg(self, t):
        self.write(t.arg)
        if t.annotation:
            self.write(": ")
            self.dispatch(t.annotation)

    # others
    def _arguments(self, t):
        first = True
        # normal arguments
        all_args = getattr(t, 'posonlyargs', []) + t.args
        defaults = [None] * (len(all_args) - len(t.defaults)) + t.defaults
        for index, elements in enumerate(zip(all_args, defaults), 1):
            a, d = elements
            if first:
                first = False
            else:
                self.write(", ")
            self.dispatch(a)
            if d:
                self.write("=")
                self.dispatch(d)
            if index == len(getattr(t, 'posonlyargs', ())):
                self.write(", /")

        # varargs, or bare '*' if no varargs but keyword-only arguments present
        if t.vararg or getattr(t, "kwonlyargs", False):
            if first:
                first = False
            else:
                self.write(", ")
            self.write("*")
            if t.vararg:
                if hasattr(t.vararg, 'arg'):
                    self.write(t.vararg.arg)
                    if t.vararg.annotation:
                        self.write(": ")
                        self.dispatch(t.vararg.annotation)
                else:
                    self.write(t.vararg)
                    if getattr(t, 'varargannotation', None):
                        self.write(": ")
                        self.dispatch(t.varargannotation)

        # keyword-only arguments
        if getattr(t, "kwonlyargs", False):
            for a, d in zip(t.kwonlyargs, t.kw_defaults):
                if first:
                    first = False
                else:
                    self.write(", ")
                self.dispatch(a),
                if d:
                    self.write("=")
                    self.dispatch(d)

        # kwargs
        if t.kwarg:
            if first:
                first = False
            else:
                self.write(", ")
            if hasattr(t.kwarg, 'arg'):
                self.write("**" + t.kwarg.arg)
                if t.kwarg.annotation:
                    self.write(": ")
                    self.dispatch(t.kwarg.annotation)
            else:
                self.write("**" + t.kwarg)
                if getattr(t, 'kwargannotation', None):
                    self.write(": ")
                    self.dispatch(t.kwargannotation)

    def _keyword(self, t):
        if t.arg is None:
            # starting from Python 3.5 this denotes a kwargs part of the invocation
            self.write("**")
        else:
            self.write(t.arg)
            self.write("=")
        self.dispatch(t.value)

    def _Lambda(self, t):
        self.write("(")
        self.write("lambda ")
        self.dispatch(t.args)
        self.write(": ")
        self.dispatch(t.body)
        self.write(")")

    def _alias(self, t):
        self.write(t.name)
        if t.asname:
            self.write(" as " + t.asname)

    def _withitem(self, t):
        self.dispatch(t.context_expr)
        if t.optional_vars:
            self.write(" as ")
            self.dispatch(t.optional_vars)


def unparse(tree):
    v = six.moves.cStringIO()
    Unparser(tree, file=v)
    return v.getvalue()


# clearml/utilities/lowlevel/file_access.py

import os
import sys
from functools import partial

import six
from typing import Optional, Any, Callable


def __buffer_writer_close_patch(self):
    self._trains_org_close()
    # noinspection PyBroadException
    try:
        self._trains_close_cb(self)
    except Exception:
        pass


def buffer_writer_close_cb(bufferwriter, callback, overwrite=False):
    # type: (Any, Callable[[Any], None], bool) -> ()
    # noinspection PyBroadException
    try:
        if not hasattr(bufferwriter, '_trains_org_close'):
            bufferwriter._trains_org_close = bufferwriter.close
            bufferwriter.close = partial(__buffer_writer_close_patch, bufferwriter)
        elif not overwrite and hasattr(bufferwriter, '_trains_close_cb'):
            return

        bufferwriter._trains_close_cb = callback
    except Exception:
        pass


def get_filename_from_file_object(file_object, flush=False, analyze_file_handle=False):
    # type: (object, bool, bool) -> Optional[str]
    """
    Return a string of the file location, extracted from any file object
    :param file_object: str, file, stream, FileIO etc.
    :param flush: If True, flush file object before returning (default: False)
    :param analyze_file_handle: If True try to retrieve filename from file handler object (default: False)
    :return: string full path of file location or None if filename cannot be extract
    """
    if isinstance(file_object, six.string_types):
        # noinspection PyBroadException
        try:
            return os.path.abspath(file_object) if file_object else file_object
        except Exception:
            return file_object
    elif hasattr(file_object, 'name'):
        filename = file_object.name
        if flush:
            # noinspection PyBroadException
            try:
                file_object.flush()
            except Exception:
                pass
        return os.path.abspath(filename)
    elif analyze_file_handle and isinstance(file_object, int) or hasattr(file_object, 'fileno'):
        # noinspection PyBroadException
        try:
            fileno = file_object if isinstance(file_object, int) else file_object.fileno()
            if sys.platform == 'win32':
                import msvcrt
                from ctypes import windll, create_string_buffer
                handle = msvcrt.get_osfhandle(fileno)
                name = create_string_buffer(2050)
                windll.kernel32.GetFinalPathNameByHandleA(handle, name, 2048, 0)
                filename = name.value.decode('utf-8')
                if filename.startswith('\\\\?\\'):
                    filename = filename[4:]
                if flush:
                    os.fsync(fileno)
                return os.path.abspath(filename)
            elif sys.platform == 'linux':
                filename = os.readlink('/proc/self/fd/{}'.format(fileno))
                if flush:
                    os.fsync(fileno)
                return os.path.abspath(filename)
            elif sys.platform == 'darwin':
                import fcntl
                name = b' ' * 1024
                # F_GETPATH = 50
                name = fcntl.fcntl(fileno, 50, name)
                filename = name.split(b'\x00')[0].decode()
                if flush:
                    os.fsync(fileno)
                return os.path.abspath(filename)
        except Exception:
            return None
    return None


# clearml/utilities/lowlevel/threads.py

import ctypes
import threading
import six
import sys
import time


def get_current_thread_id():
    return threading._get_ident() if six.PY2 else threading.get_ident()  # noqa


# Nasty hack to raise exception for other threads
def _lowlevel_async_raise(thread_obj, exception=None):
    NULL = 0  # noqa
    found = False
    target_tid = 0
    for tid, tobj in threading._active.items():  # noqa
        if tobj is thread_obj:
            found = True
            target_tid = tid
            break

    if not found:
        # raise ValueError("Invalid thread object")
        return False

    if not exception:
        exception = SystemExit()

    if sys.version_info.major >= 3 and sys.version_info.minor >= 7:
        target_tid = ctypes.c_ulong(target_tid)
        NULL = ctypes.c_ulong(NULL)  # noqa
    else:
        target_tid = ctypes.c_long(target_tid)
        NULL = ctypes.c_long(NULL)  # noqa

    # noinspection PyBroadException
    try:
        ret = ctypes.pythonapi.PyThreadState_SetAsyncExc(target_tid, ctypes.py_object(exception))
    except Exception:
        ret = 0

    # ref: http://docs.python.org/c-api/init.html#PyThreadState_SetAsyncExc
    if ret == 0:
        # raise ValueError("Invalid thread ID")
        return False
    elif ret > 1:
        # Huh? Why would we notify more than one threads?
        # Because we punch a hole into C level interpreter.
        # So it is better to clean up the mess.
        # noinspection PyBroadException
        try:
            ctypes.pythonapi.PyThreadState_SetAsyncExc(target_tid, NULL)
        except Exception:
            pass
        # raise SystemError("PyThreadState_SetAsyncExc failed")
        return False

    # print("Successfully set asynchronized exception for", target_tid)
    return True


def kill_thread(thread_obj, wait=False):
    if not _lowlevel_async_raise(thread_obj, SystemExit()):
        return False

    while wait and thread_obj.is_alive():
        time.sleep(0.1)
    return True


def __wait_thread(a_thread, a_event):
    # noinspection PyBroadException
    try:
        a_thread.join()
        a_event.set()
    except Exception:
        pass


def threadpool_waited_join(thread_object, timeout):
    """
    Call threadpool.join() with timeout. If join completed return True, otherwise False
    Notice: This function creates another daemon thread and kills it, use with care.

    :param thread_object: Thread to join
    :param float timeout: timeout in seconds for the join operation to complete
    :return: True os join() completed
    """
    if not thread_object:
        return True

    if isinstance(thread_object, threading.Thread):
        thread_object.join(timeout=timeout)
        return not thread_object.is_alive()

    done_signal = threading.Event()
    waitable = threading.Thread(target=__wait_thread, args=(thread_object, done_signal,))
    waitable.daemon = True
    waitable.start()

    if not done_signal.wait(timeout=timeout):
        kill_thread(waitable)
        return False
    return True


if __name__ == '__main__':
    def demo_thread(*_, **__):
        from time import sleep
        for i in range(5):
            print('.')
            sleep(1.)

    t = threading.Thread(target=demo_thread)
    t.daemon = True
    t.start()
    print(threadpool_waited_join(t, 2.0))


# clearml/utilities/pigar/__init__.py

# -*- coding: utf-8 -*-


# clearml/utilities/pigar/__main__.py

# -*- coding: utf-8 -*-

from __future__ import print_function, division, absolute_import

import os
import codecs

import six

from .reqs import project_import_modules, is_std_or_local_lib, is_base_module
from .utils import lines_diff
from .log import logger
from .modules import ReqsModules


class GenerateReqs(object):
    _force_modules_reqs = dict()

    def __init__(self, save_path, project_path, ignores,
                 installed_pkgs, comparison_operator='=='):
        self._save_path = save_path
        self._project_path = project_path
        self._ignores = ignores
        self._installed_pkgs = installed_pkgs
        self._maybe_local_mods = set()
        self._local_mods = dict()
        self._relative_imports = set()
        self._comparison_operator = comparison_operator

    def extract_reqs(self, module_callback=None, entry_point_filename=None):
        """Extract requirements from project."""

        reqs = ReqsModules()
        guess = ReqsModules()
        local = ReqsModules()

        num_local_mod = 0
        if self.__module__:
            # create a copy, do not change the class set
            our_module = self.__module__.split('.')[0]
            if our_module and our_module not in self._force_modules_reqs:
                from ...version import __version__
                self._force_modules_reqs[our_module] = __version__

        # make the entry point absolute (relative to the root path)
        if entry_point_filename and not os.path.isabs(entry_point_filename):
            entry_point_filename = os.path.join(self._project_path, entry_point_filename) \
                if os.path.isdir(self._project_path) else None

        # check if the entry point script is self contained, i.e. does not use the rest of the project
        if entry_point_filename and os.path.isfile(entry_point_filename) and not self._local_mods:
            modules, try_imports, local_mods = project_import_modules(entry_point_filename, self._ignores)
            if not local_mods:
                # update the self._local_mods
                self._filter_modules(modules, local_mods)
                # check how many local modules we have, excluding ourselves
                num_local_mod = len(set(self._local_mods.keys()) - set(self._force_modules_reqs.keys()))

            # if we have any module/package we cannot find, take no chances and scan the entire project
            # if we have local modules and they are not just us.
            if num_local_mod or local_mods or self._relative_imports:
                modules, try_imports, local_mods = project_import_modules(
                    self._project_path, self._ignores)

        else:
            modules, try_imports, local_mods = project_import_modules(
                self._project_path, self._ignores)

        if module_callback:
            modules = module_callback(modules)

        # Filtering modules
        candidates = self._filter_modules(modules, local_mods)

        # make sure we are in candidates
        candidates |= set(self._force_modules_reqs.keys())

        logger.info('Check module in local environment.')
        reqs_module_name = []
        for name in candidates:
            logger.info('Checking module: %s', name)
            if name in self._installed_pkgs:
                pkg_name, version = self._installed_pkgs[name]
                if name not in modules:
                    modules.add(name, name, 0)
                if not version and pkg_name and pkg_name.startswith('-e '):
                    reqs.add('{} @ {}'.format(name, pkg_name.replace('-e ', '', 1)), version, modules[name])
                else:
                    reqs.add(pkg_name, version, modules[name])
                reqs_module_name.append(name)
            elif name in modules:
                guess.add(name, 0, modules[name])

        # add local modules, so we know what is used but not installed.
        project_path = os.path.realpath(self._project_path)
        for name in self._local_mods:
            if name in modules and name not in reqs_module_name:
                if name in self._force_modules_reqs:
                    reqs.add(name, self._force_modules_reqs[name], modules[name])
                    reqs_module_name.append(name)
                    continue

                # if this is a base module, we have it in installed modules but package name is None
                mod_path = os.path.realpath(self._local_mods[name])
                if is_base_module(mod_path):
                    continue

                # if this is a folder of our project, we can safely ignore it
                if (six.PY3 and os.path.commonpath([project_path]) == os.path.commonpath([project_path, mod_path])) or \
                        (six.PY2 and
                         os.path.commonprefix([project_path]) == os.path.commonprefix([project_path, mod_path])):
                    continue

                relpath = os.path.relpath(self._local_mods[name], self._project_path)
                if not relpath.startswith('.'):
                    relpath = '.' + os.path.sep + relpath
                local.add(name, relpath, modules[name])

        return reqs, try_imports, guess, local

    @classmethod
    def get_forced_modules(cls):
        return cls._force_modules_reqs

    @classmethod
    def add_forced_module(cls, module_name, module_version):
        cls._force_modules_reqs[module_name] = module_version

    def _write_reqs(self, reqs):
        print('Writing requirements to "{0}"'.format(
            self._save_path))
        with open(self._save_path, 'w+') as f:
            f.write('# Requirements automatically generated by pigar.\n'
                    '# https://github.com/damnever/pigar\n')
            for k, v in reqs.sorted_items():
                f.write('\n')
                f.write(''.join(['# {0}\n'.format(c)
                                 for c in v.comments.sorted_items()]))
                if k == '-e':
                    f.write('{0} {1}\n'.format(k, v.version))
                elif v:
                    f.write('{0} {1} {2}\n'.format(
                        k, self._comparison_operator, v.version))
                else:
                    f.write('{0}\n'.format(k))

    def _best_matchs(self, name, pkgs):
        # If imported name equals to package name.
        if name in pkgs:
            return [pkgs[pkgs.index(name)]]
        # If not, return all possible packages.
        return pkgs

    def _filter_modules(self, modules, local_mods):
        candidates = set()

        logger.info('Filtering modules ...')
        for module in modules:
            logger.info('Checking module: %s', module)
            if not module:
                continue
            if module.startswith('.'):
                self._relative_imports.add(module)
                continue
            if module in local_mods:
                self._maybe_local_mods.add(module)
            module_std_local = is_std_or_local_lib(module)
            if module_std_local is True:
                continue
            if isinstance(module_std_local, str):
                self._local_mods[module] = module_std_local
                continue
            candidates.add(module)

        return candidates

    def _invalid_reqs(self, reqs):
        for name, detail in reqs.sorted_items():
            print(
                '  {0} referenced from:\n    {1}'.format(
                    name,
                    '\n    '.join(detail.comments.sorted_items())
                )
            )

    def _save_old_reqs(self):
        if os.path.isfile(self._save_path):
            with codecs.open(self._save_path, 'rb', 'utf-8') as f:
                self._old_reqs = f.readlines()

    def _reqs_diff(self):
        if not hasattr(self, '_old_reqs'):
            return
        with codecs.open(self._save_path, 'rb', 'utf-8') as f:
            new_reqs = f.readlines()
        is_diff, diffs = lines_diff(self._old_reqs, new_reqs)
        msg = 'Requirements file has been covered, '
        if is_diff:
            msg += 'there is the difference:'
            print('{0}\n{1}'.format(msg, ''.join(diffs)), end='')
        else:
            msg += 'no difference.'
            print(msg)


# clearml/utilities/pigar/log.py

# -*- coding: utf-8 -*-

from __future__ import print_function, division, absolute_import

import logging.handlers


logger = logging.getLogger('pigar')
logger.setLevel(logging.WARNING)


# clearml/utilities/pigar/modules.py

# -*- coding: utf-8 -*-

from __future__ import print_function, division, absolute_import

import collections


# FIXME: Just a workaround, not a radical cure..
_special_cases = {
    "dogpile.cache": "dogpile.cache",
    "dogpile.core": "dogpile.core",
    "ruamel.yaml": "ruamel.yaml",
    "ruamel.ordereddict": "ruamel.ordereddict",
}


class Modules(dict):
    """Modules object will be used to store modules information."""

    def __init__(self):
        super(Modules, self).__init__()


class ImportedModules(Modules):

    def __init__(self):
        super(ImportedModules, self).__init__()

    def add(self, name, file, lineno):
        if name is None:
            return

        names = list()
        special_name = '.'.join(name.split('.')[:2])
        # Flask extension.
        if name.startswith('flask.ext.'):
            names.append('flask')
            names.append('flask_' + name.split('.')[2])
        # Special cases..
        elif special_name in _special_cases:
            names.append(_special_cases[special_name])
        # Other.
        elif '.' in name and not name.startswith('.'):
            names.append(name.split('.')[0])
        else:
            names.append(name)

        for nm in names:
            if nm not in self:
                self[nm] = _Locations()
            self[nm].add(file, lineno)

    def __or__(self, obj):
        for name, locations in obj.items():
            for file, linenos in locations.items():
                for lineno in linenos:
                    self.add(name, file, lineno)
        return self


class ReqsModules(Modules):

    _Detail = collections.namedtuple('Detail', ['version', 'comments'])

    def __init__(self):
        super(ReqsModules, self).__init__()
        self._sorted = None

    def add(self, package, version, locations):
        if package in self:
            self[package].comments.extend(locations)
        else:
            self[package] = self._Detail(version, locations)

    def sorted_items(self):
        if self._sorted is None:
            self._sorted = sorted(self.items())
        return self._sorted

    def remove(self, *names):
        for name in names:
            if name in self:
                self.pop(name)
        self._sorted = None


class _Locations(dict):
    """_Locations store code locations(file, linenos)."""

    def __init__(self):
        super(_Locations, self).__init__()
        self._sorted = None

    def add(self, file, lineno):
        if file in self and lineno not in self[file]:
            self[file].append(lineno)
        else:
            self[file] = [lineno]

    def extend(self, obj):
        for file, linenos in obj.items():
            for lineno in linenos:
                self.add(file, lineno)

    def sorted_items(self):
        if self._sorted is None:
            self._sorted = [
                '{0}: {1}'.format(f, ','.join([str(n) for n in sorted(ls)]))
                for f, ls in sorted(self.items())
            ]
        return self._sorted


# clearml/utilities/pigar/reqs.py

# -*- coding: utf-8 -*-

from __future__ import print_function, division, absolute_import

import json
import os
import sys
import six
import fnmatch
import importlib
import ast
import doctest
import collections
import functools
from pathlib2 import Path
from .utils import parse_git_config
from .modules import ImportedModules


def project_import_modules(project_path, ignores):
    """
    copied form pigar req.project_import_modules patching, os.getcwd() is incorrectly used
    """
    modules = ImportedModules()
    try_imports = set()
    local_mods = list()
    ignore_paths = set()
    venv_subdirs = set(('bin', 'etc', 'include', 'lib', 'lib64', 'share'))
    ignore_absolute = []
    if not ignores:
        ignore_paths.add('.git')
    else:
        for path in ignores:
            ignore_paths.add(Path(path).name)

    if os.path.isfile(project_path):
        try:
            fake_path = Path(project_path).name
            with open(project_path, 'rb') as f:
                fmodules, try_ipts = file_import_modules(fake_path, f.read())
                modules |= fmodules
                try_imports |= try_ipts
        except Exception:
            pass
    else:
        cur_dir = project_path
        for dirpath, dirnames, files in os.walk(os.path.abspath(project_path), followlinks=True):
            # see if we have a parent folder in the ignore list
            if ignore_paths & set(Path(dirpath).relative_to(cur_dir).parts):
                continue
            # check if we are in a subfolder of ignore list
            if any(True for prefix in ignore_absolute if dirpath.startswith(prefix)):
                continue
            # Hack detect if this is a virtual-env folder, if so add it to the uignore list
            if set(dirnames) == venv_subdirs:
                ignore_absolute.append(Path(dirpath).as_posix() + os.sep)
                continue

            py_files = list()
            for fn in files:
                # C extension.
                if fn.endswith('.so'):
                    local_mods.append(fn[:-3])
                # Normal Python file.
                if fn.endswith('.py'):
                    local_mods.append(fn[:-3])
                    py_files.append(fn)
            if '__init__.py' in files:
                local_mods.append(os.path.basename(dirpath))
            for file in py_files:
                try:
                    fpath = os.path.join(dirpath, file)
                    fake_path = Path(fpath).relative_to(cur_dir).as_posix()
                    with open(fpath, 'rb') as f:
                        fmodules, try_ipts = file_import_modules(fake_path, f.read())
                        modules |= fmodules
                        try_imports |= try_ipts
                except Exception:
                    pass

    return modules, try_imports, local_mods


def file_import_modules(fpath, fdata):
    """Get single file all imported modules."""
    modules = ImportedModules()
    str_codes = collections.deque([(fdata, 1)])
    try_imports = set()

    while str_codes:
        str_code, lineno = str_codes.popleft()
        ic = ImportChecker(fpath, lineno)
        try:
            parsed = ast.parse(str_code)
            ic.visit(parsed)
        # Ignore SyntaxError in Python code.
        except SyntaxError:
            pass
        modules |= ic.modules
        str_codes.extend(ic.str_codes)
        try_imports |= ic.try_imports
        del ic

    return modules, try_imports


class ImportChecker(object):

    def __init__(self, fpath, lineno):
        self._fpath = fpath
        self._lineno = lineno - 1
        self._modules = ImportedModules()
        self._str_codes = collections.deque()
        self._try_imports = set()

    def visit_Import(self, node, try_=False):
        """As we know: `import a [as b]`."""
        lineno = node.lineno + self._lineno
        for alias in node.names:
            self._modules.add(alias.name, self._fpath, lineno)
            if try_:
                self._try_imports.add(alias.name)

    def visit_ImportFrom(self, node, try_=False):
        """
        As we know: `from a import b [as c]`. If node.level is not 0,
        import statement like this `from .a import b`.
        """
        mod_name = node.module
        level = node.level
        if mod_name is None:
            level -= 1
            mod_name = ''
        for alias in node.names:
            name = level * '.' + mod_name + '.' + alias.name
            self._modules.add(name, self._fpath, node.lineno + self._lineno)
            if try_:
                self._try_imports.add(name)

    def visit_TryExcept(self, node):
        """
        If modules which imported by `try except` and not found,
        maybe them come from other Python version.
        """
        for ipt in node.body:
            if ipt.__class__.__name__.startswith('Import'):
                method = 'visit_' + ipt.__class__.__name__
                getattr(self, method)(ipt, True)
        for handler in node.handlers:
            for ipt in handler.body:
                if ipt.__class__.__name__.startswith('Import'):
                    method = 'visit_' + ipt.__class__.__name__
                    getattr(self, method)(ipt, True)

    # For Python 3.3+
    visit_Try = visit_TryExcept

    def visit_Exec(self, node):
        """
        Check `expression` of `exec(expression[, globals[, locals]])`.
        **Just available in python 2.**
        """
        if hasattr(node.body, 's'):
            self._str_codes.append((node.body.s, node.lineno + self._lineno))
        # PR#13: https://github.com/damnever/pigar/pull/13
        # Sometimes exec statement may be called with tuple in Py2.7.6
        elif hasattr(node.body, 'elts') and len(node.body.elts) >= 1:
            self._str_codes.append(
                (node.body.elts[0].s, node.lineno + self._lineno))

    def visit_Expr(self, node):
        """
        Check `expression` of `eval(expression[, globals[, locals]])`.
        Check `expression` of `exec(expression[, globals[, locals]])`
        in python 3.
        Check `name` of `__import__(name[, globals[, locals[,
        fromlist[, level]]]])`.
        Check `name` or `package` of `importlib.import_module(name,
        package=None)`.
        """
        # Built-in functions
        value = node.value
        if isinstance(value, ast.Call):
            if hasattr(value.func, 'id'):
                if (value.func.id == 'eval' and
                        hasattr(node.value.args[0], 's')):
                    self._str_codes.append(
                        (node.value.args[0].s, node.lineno + self._lineno))
                # **`exec` function in Python 3.**
                elif (value.func.id == 'exec' and
                        hasattr(node.value.args[0], 's')):
                    self._str_codes.append(
                        (node.value.args[0].s, node.lineno + self._lineno))
                # `__import__` function.
                elif (value.func.id == '__import__' and
                        len(node.value.args) > 0 and
                        hasattr(node.value.args[0], 's')):
                    self._modules.add(node.value.args[0].s, self._fpath,
                                      node.lineno + self._lineno)
            # `import_module` function.
            elif getattr(value.func, 'attr', '') == 'import_module':
                module = getattr(value.func, 'value', None)
                if (module is not None and
                        getattr(module, 'id', '') == 'importlib'):
                    args = node.value.args
                    arg_len = len(args)
                    if arg_len > 0 and hasattr(args[0], 's'):
                        name = args[0].s
                        if not name.startswith('.'):
                            self._modules.add(name, self._fpath,
                                              node.lineno + self._lineno)
                        elif arg_len == 2 and hasattr(args[1], 's'):
                            self._modules.add(args[1].s, self._fpath,
                                              node.lineno + self._lineno)

    def visit_FunctionDef(self, node):
        """
        Check docstring of function, if docstring is used for doctest.
        """
        docstring = self._parse_docstring(node)
        if docstring:
            self._str_codes.append((docstring, node.lineno + self._lineno + 2))

    def visit_ClassDef(self, node):
        """
        Check docstring of class, if docstring is used for doctest.
        """
        docstring = self._parse_docstring(node)
        if docstring:
            self._str_codes.append((docstring, node.lineno + self._lineno + 2))

    def visit(self, node):
        """Visit a node, no recursively."""
        for node in ast.walk(node):
            method = 'visit_' + node.__class__.__name__
            getattr(self, method, lambda x: x)(node)

    @staticmethod
    def _parse_docstring(node):
        """Extract code from docstring."""
        docstring = ast.get_docstring(node)
        if docstring:
            parser = doctest.DocTestParser()
            try:
                dt = parser.get_doctest(docstring, {}, None, None, None)
            except ValueError:
                # >>> 'abc'
                pass
            else:
                examples = dt.examples
                return '\n'.join([example.source for example in examples])
        return None

    @property
    def modules(self):
        return self._modules

    @property
    def str_codes(self):
        return self._str_codes

    @property
    def try_imports(self):
        return set((name.split('.')[0] if name and '.' in name else name)
                   for name in self._try_imports)


def _checked_cache(func):
    checked = dict()

    @functools.wraps(func)
    def _wrapper(name):
        if name not in checked:
            checked[name] = func(name)
        return checked[name]

    return _wrapper


@_checked_cache
def is_std_or_local_lib(name):
    """Check whether it is stdlib module.
    True if std lib
    False if installed package
    str if local library
    """
    exist = True
    if six.PY2:
        import imp  # noqa
        from types import FileType  # noqa
        module_info = ('', '', '')
        try:
            module_info = imp.find_module(name)
        except ImportError:
            try:
                # __import__(name)
                importlib.import_module(name)
                module_info = imp.find_module(name)
                sys.modules.pop(name)
            except ImportError:
                exist = False
        # Testcase: ResourceWarning
        if isinstance(module_info[0], FileType):
            module_info[0].close()  # noqa
        mpath = module_info[1]  # noqa
        # make sure we remove built-in modules
        if mpath and not os.path.exists(mpath):
            mpath = None
    else:
        module_info = None
        try:
            module_info = importlib.util.find_spec(name) # noqa
        except ImportError:
            return False
        except ValueError:
            # if we got here, the loader failed on us, meaning this is definitely a module and not std
            return False
        if not module_info:
            if name == '__builtin__':
                return True
            return False
        mpath = module_info.origin
        # this is std
        if mpath == 'built-in':
            mpath = None

    if exist and mpath is not None:
        if ('site-packages' in mpath or
                'dist-packages' in mpath or
                'bin/' in mpath and mpath.endswith('.py')):
            exist = False
        elif ((sys.prefix not in mpath) and
              (six.PY2 or (sys.base_exec_prefix not in mpath)) and
              (six.PY2 or (sys.base_prefix not in mpath))):
            exist = mpath

    return exist


def get_installed_pkgs_detail():
    """
    HACK: bugfix of the original pigar get_installed_pkgs_detail

    Get mapping for import top level name
    and install package name with version.
    """
    mapping = dict()

    for path in sys.path:
        if os.path.isdir(path) and path.rstrip('/').endswith(
                ('site-packages', 'dist-packages')):
            new_mapping = _search_path(path)
            # BUGFIX:
            # override with previous, just like python resolves imports, the first match is the one used.
            # unlike the original implementation, where the last one is used.
            new_mapping.update(mapping)
            mapping = new_mapping

    # HACK: prefer tensorflow_gpu over tensorflow
    if 'tensorflow_gpu' in mapping:
        mapping['tensorflow'] = mapping['tensorflow_gpu']

    return mapping


def is_base_module(module_path):
    python_base = '{}python{}.{}'.format(os.sep, sys.version_info.major, sys.version_info.minor)
    for path in sys.path:
        if os.path.isdir(path) and path.rstrip('/').endswith(
                (python_base, )):
            if not path[-1] == os.sep:
                path += os.sep
            if module_path.startswith(path):
                return True
    return False


def _search_path(path):
    mapping = dict()

    for file in os.listdir(path):
        # Install from PYPI.
        if fnmatch.fnmatch(file, '*-info'):
            pkg_name, version = file.split('-')[:2]
            if version.endswith('dist'):
                version = version.rsplit('.', 1)[0]
            # Issue for ubuntu: sudo pip install xxx
            elif version.endswith('egg'):
                version = version.rsplit('.', 1)[0]

            mapping_pkg_name = pkg_name
            # pep610 support. add support for new pip>=20.1 git reference feature
            git_direct_json = os.path.join(path, file, 'direct_url.json')
            if os.path.isfile(git_direct_json):
                # noinspection PyBroadException
                try:
                    with open(git_direct_json, 'r') as f:
                        vcs_info = json.load(f)

                    if 'vcs_info' in vcs_info:
                        git_url = '{vcs}+{url}@{commit}#egg={package}'.format(
                            vcs=vcs_info['vcs_info']['vcs'], url=vcs_info['url'],
                            commit=vcs_info['vcs_info']['commit_id'], package=pkg_name)
                        # Bugfix: package name should be the URL link, because we need it unique
                        # mapping[pkg_name] = ('-e', git_url)
                        pkg_name, version = '-e {}'.format(git_url), ''
                    elif 'url' in vcs_info:
                        url_link = vcs_info.get('url', '').strip().lower()
                        if url_link and not url_link.startswith('file://'):
                            pkg_name, version = vcs_info['url'], ''

                except Exception:
                    pass

            # default
            mapping[mapping_pkg_name] = (pkg_name, version)

            # analyze 'top_level.txt' if it exists
            top_level = os.path.join(path, file, 'top_level.txt')
            if not os.path.isfile(top_level):
                continue
            with open(top_level, 'r') as f:
                for line in f:
                    mapping[line.strip()] = (pkg_name, version)

        # Install from local and available in GitHub.
        elif fnmatch.fnmatch(file, '*-link'):
            link = os.path.join(path, file)
            if not os.path.isfile(link):
                continue
            # Link path.
            with open(link, 'r') as f:
                for line in f:
                    line = line.strip()
                    if line != '.':
                        dev_dir = line
            if not dev_dir:
                continue
            if not os.path.exists(dev_dir):
                continue
            # Egg info path.
            info_dir = [_file for _file in os.listdir(dev_dir)
                        if _file.endswith('egg-info')]
            if not info_dir:
                continue
            info_dir = info_dir[0]
            top_level = os.path.join(dev_dir, info_dir, 'top_level.txt')
            # Check whether it can be imported.
            if not os.path.isfile(top_level):
                continue

            # Check .git dir.
            git_path = os.path.join(dev_dir, '.git')
            if os.path.isdir(git_path):
                config = parse_git_config(git_path)
                url = config.get('remote "origin"', {}).get('url')
                if not url:
                    continue
                branch = 'branch "master"'
                if branch not in config:
                    for section in config:
                        if 'branch' in section:
                            branch = section
                            break
                if not branch:
                    continue
                branch = branch.split()[1][1:-1]

                pkg_name = info_dir.split('.egg')[0]
                git_url = 'git+{0}@{1}#egg={2}'.format(url, branch, pkg_name)
                with open(top_level, 'r') as f:
                    for line in f:
                        # Bugfix: package name should be the URL link, because we need it unique
                        # mapping[line.strip()] = ('-e', git_url)
                        mapping[line.strip()] = ('-e {}'.format(git_url), '')

    return mapping


# clearml/utilities/pigar/unpack.py

# -*- coding: utf-8 -*-

from __future__ import print_function, division, absolute_import

import tarfile
import zipfile
import re
import string
import io


class Archive(object):
    """Archive provides a consistent interface for unpacking
    compressed file.
    """

    def __init__(self, filename, fileobj):
        self._filename = filename
        self._fileobj = fileobj
        self._file = None
        self._names = None
        self._read = None

    @property
    def filename(self):
        return self._filename

    @property
    def names(self):
        """If name list is not required, do not get it."""
        if self._file is None:
            self._prepare()
        if not hasattr(self, '_namelist'):
            self._namelist = self._names()
        return self._namelist

    def close(self):
        """Close file object."""
        if self._file is not None:
            self._file.close()
        if hasattr(self, '_namelist'):
            del self._namelist
        self._filename = self._fileobj = None
        self._file = self._names = self._read = None

    def read(self, filename):
        """Read one file from archive."""
        if self._file is None:
            self._prepare()
        return self._read(filename)

    def unpack(self, to_path):
        """Unpack compressed files to path."""
        if self._file is None:
            self._prepare()
        self._safe_extractall(to_path)

    def _prepare(self):
        if self._filename.endswith(('.tar.gz', '.tar.bz2', '.tar.xz')):
            self._prepare_tarball()
        # An .egg file is actually just a .zip file
        # with a different extension, .whl too.
        elif self._filename.endswith(('.zip', '.egg', '.whl')):
            self._prepare_zip()
        else:
            raise ValueError("unreadable: {0}".format(self._filename))

    def _safe_extractall(self, to_path='.'):
        unsafe = []
        for name in self.names:
            if not self.is_safe(name):
                unsafe.append(name)
        if unsafe:
            raise ValueError("unsafe to unpack: {}".format(unsafe))
        self._file.extractall(to_path)

    def _prepare_zip(self):
        self._file = zipfile.ZipFile(self._fileobj)
        self._names = self._file.namelist
        self._read = self._file.read

    def _prepare_tarball(self):
        # tarfile has no read method
        def _read(filename):
            f = self._file.extractfile(filename)
            return f.read()

        self._file = tarfile.open(mode='r:*', fileobj=self._fileobj)
        self._names = self._file.getnames
        self._read = _read

    def is_safe(self, filename):
        return not (filename.startswith(("/", "\\")) or
                    (len(filename) > 1 and filename[1] == ":" and
                     filename[0] in string.ascii_letter) or
                    re.search(r"[.][.][/\\]", filename))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()


def top_level(url, data):
    """Read top level names from compressed file."""
    sb = io.BytesIO(data)
    txt = None
    with Archive(url, sb) as archive:
        file = None
        for name in archive.names:
            if name.lower().endswith('top_level.txt'):
                file = name
                break
        if file:
            txt = archive.read(file).decode('utf-8')
    sb.close()
    return [name.replace('/', '.') for name in txt.splitlines()] if txt else []


# clearml/utilities/pigar/utils.py

# -*- coding: utf-8 -*-

from __future__ import print_function, division, absolute_import

import os
import re
import sys
import difflib


PY32 = sys.version_info[:2] == (3, 2)

if sys.version_info[0] == 3:
    binary_type = bytes
else:
    binary_type = str


class Dict(dict):
    """Convert dict key object to attribute."""

    def __init__(self, *args, **kwargs):
        super(Dict, self).__init__(*args, **kwargs)

    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            raise AttributeError('"{0}"'.format(name))

    def __setattr__(self, name, value):
        self[name] = value


def parse_reqs(fpath):
    pkg_v_re = re.compile(r'^(?P<pkg>[^><==]+)[><==]{,2}(?P<version>.*)$')
    """Parse requirements file."""
    reqs = dict()
    with open(fpath, 'r') as f:
        for line in f:
            if line.startswith('#'):
                continue
            m = pkg_v_re.match(line.strip())
            if m:
                d = m.groupdict()
                reqs[d['pkg'].strip()] = d['version'].strip()
    return reqs


def cmp_to_key(cmp_func):
    """Convert a cmp=fcuntion into a key=function."""
    class K(object):
        def __init__(self, obj, *args):
            self.obj = obj

        def __lt__(self, other):
            return cmp_func(self.obj, other.obj) < 0

        def __gt__(self, other):
            return cmp_func(self.obj, other.obj) > 0

        def __eq__(self, other):
            return cmp_func(self.obj, other.obj) == 0

    return K


def compare_version(version1, version2):
    """Compare version number, such as 1.1.1 and 1.1b2.0."""
    v1, v2 = list(), list()

    for item in version1.split('.'):
        if item.isdigit():
            v1.append(int(item))
        else:
            v1.extend([i for i in _group_alnum(item)])
    for item in version2.split('.'):
        if item.isdigit():
            v2.append(int(item))
        else:
            v2.extend([i for i in _group_alnum(item)])

    while v1 and v2:
        item1, item2 = v1.pop(0), v2.pop(0)
        if item1 > item2:
            return 1
        elif item1 < item2:
            return -1

    if v1:
        return 1
    elif v2:
        return -1
    return 0


def _group_alnum(s):
    tmp = list()
    flag = 1 if s[0].isdigit() else 0
    for c in s:
        if c.isdigit():
            if flag == 0:
                yield ''.join(tmp)
                tmp = list()
                flag = 1
            tmp.append(c)
        elif c.isalpha():
            if flag == 1:
                yield int(''.join(tmp))
                tmp = list()
                flag = 0
            tmp.append(c)
    last = ''.join(tmp)
    yield (int(last) if flag else last)


def parse_git_config(path):
    """Parse git config file."""
    config = dict()
    section = None

    with open(os.path.join(path, 'config'), 'r') as f:
        for line in f:
            line = line.strip()
            if line.startswith('['):
                section = line[1: -1].strip()
                config[section] = dict()
            elif section:
                key, value = line.replace(' ', '').split('=')
                config[section][key] = value
    return config


def lines_diff(lines1, lines2):
    """Show difference between lines."""
    is_diff = False
    diffs = list()

    for line in difflib.ndiff(lines1, lines2):
        if not is_diff and line[0] in ('+', '-'):
            is_diff = True
        diffs.append(line)

    return is_diff, diffs


# clearml/utilities/plotlympl/__init__.py

"""
matplotlylib
============

This module converts matplotlib figure objects into JSON structures which can
be understood and visualized by Plotly.

Most of the functionality should be accessed through the parent directory's
'tools' module or 'plotly' package.

"""
from __future__ import absolute_import

from .renderer import PlotlyRenderer
from .mplexporter import Exporter

__all__ = [
    "PlotlyRenderer",
    "Exporter"
]


# clearml/utilities/plotlympl/mpltools.py

"""
Tools

A module for converting from mpl language to plotly language.

"""
import math

import warnings
import matplotlib.dates
from matplotlib import __version__ as matplotlib_version
try:
    from matplotlib.patches import FancyBboxPatch
except ImportError:
    FancyBboxPatch = None


def check_maplotlib_max_version(ver_maj, ver_min):
    """
    return True if check_maplotlib_version is lower than specified version
    """
    parts = matplotlib_version.split('.')
    if not parts:
        return False
    if parts[0] < str(ver_maj):
        return True
    if len(parts) > 1 and parts[1] < str(ver_min):
        return True
    return False


def check_bar_match(old_bar, new_bar):
    """Check if two bars belong in the same collection (bar chart).

    Positional arguments:
    old_bar -- a previously sorted bar dictionary.
    new_bar -- a new bar dictionary that needs to be sorted.

    """
    tests = []
    tests += (new_bar["orientation"] == old_bar["orientation"],)
    tests += (new_bar["facecolor"] == old_bar["facecolor"],)
    if new_bar["orientation"] == "v":
        new_width = new_bar["x1"] - new_bar["x0"]
        old_width = old_bar["x1"] - old_bar["x0"]
        tests += (new_width - old_width < 0.000001,)
        tests += (new_bar["y0"] == old_bar["y0"],)
    elif new_bar["orientation"] == "h":
        new_height = new_bar["y1"] - new_bar["y0"]
        old_height = old_bar["y1"] - old_bar["y0"]
        tests += (new_height - old_height < 0.000001,)
        tests += (new_bar["x0"] == old_bar["x0"],)
    if all(tests):
        return True
    else:
        return False


def check_corners(inner_obj, outer_obj):
    inner_corners = inner_obj.get_window_extent().corners()
    outer_corners = outer_obj.get_window_extent().corners()
    if inner_corners[0][0] < outer_corners[0][0]:
        return False
    elif inner_corners[0][1] < outer_corners[0][1]:
        return False
    elif inner_corners[3][0] > outer_corners[3][0]:
        return False
    elif inner_corners[3][1] > outer_corners[3][1]:
        return False
    else:
        return True


def convert_dash(mpl_dash):
    """Convert mpl line symbol to plotly line symbol and return symbol."""
    if mpl_dash in DASH_MAP:
        return DASH_MAP[mpl_dash]
    else:
        dash_array = mpl_dash.split(",")

        if len(dash_array) < 2:
            return "solid"

        # Catch the exception where the off length is zero, in case
        # matplotlib 'solid' changes from '10,0' to 'N,0'
        if math.isclose(float(dash_array[1]), 0.0):
            return "solid"

        # If we can't find the dash pattern in the map, convert it
        # into custom values in px, e.g. '7,5' -> '7px,5px'
        dashpx = ",".join([x + "px" for x in dash_array])

        # TODO: rewrite the convert_dash code
        # only strings 'solid', 'dashed', etc allowed
        if dashpx == "7.4px,3.2px":
            dashpx = "dashed"
        elif dashpx == "12.8px,3.2px,2.0px,3.2px":
            dashpx = "dashdot"
        elif dashpx == "2.0px,3.3px":
            dashpx = "dotted"
        return dashpx


def convert_path(path):
    verts = path[0]  # noqa: F841,  may use this later
    code = tuple(path[1])
    if code in PATH_MAP:
        return PATH_MAP[code]
    else:
        return None


def convert_symbol(mpl_symbol):
    """Convert mpl marker symbol to plotly symbol and return symbol."""
    if isinstance(mpl_symbol, list):
        symbol = list()
        for s in mpl_symbol:
            symbol += [convert_symbol(s)]
        return symbol
    elif mpl_symbol in SYMBOL_MAP:
        return SYMBOL_MAP[mpl_symbol]
    else:
        return "circle"  # default


def hex_to_rgb(value):
    """
    Change a hex color to an rgb tuple

    :param (str|unicode) value: The hex string we want to convert.
    :return: (int, int, int) The red, green, blue int-tuple.

    Example:

        '#FFFFFF' --> (255, 255, 255)

    """
    value = value.lstrip("#")
    lv = len(value)
    return tuple(int(value[i: i + lv // 3], 16) for i in range(0, lv, lv // 3))


def merge_color_and_opacity(color, opacity):
    """
    Merge hex color with an alpha (opacity) to get an rgba tuple.

    :param (str|unicode) color: A hex color string.
    :param (float|int) opacity: A value [0, 1] for the 'a' in 'rgba'.
    :return: (int, int, int, float) The rgba color and alpha tuple.

    """
    if color is None:  # None can be used as a placeholder, just bail.
        return None

    rgb_tup = hex_to_rgb(color)
    if opacity is None:
        return "rgb {}".format(rgb_tup)

    opacity = int(opacity*255.) if opacity <= 1.0 else int(opacity)
    rgba_tup = rgb_tup + (opacity,)
    return "rgba {}".format(rgba_tup)


def convert_va(mpl_va):
    """Convert mpl vertical alignment word to equivalent HTML word.

    Text alignment specifiers from mpl differ very slightly from those used
    in HTML. See the VA_MAP for more details.

    Positional arguments:
    mpl_va -- vertical mpl text alignment spec.

    """
    if mpl_va in VA_MAP:
        return VA_MAP[mpl_va]
    else:
        return None  # let plotly figure it out!


def convert_x_domain(mpl_plot_bounds, mpl_max_x_bounds):
    """Map x dimension of current plot to plotly's domain space.

    The bbox used to locate an axes object in mpl differs from the
    method used to locate axes in plotly. The mpl version locates each
    axes in the figure so that axes in a single-plot figure might have
    the bounds, [0.125, 0.125, 0.775, 0.775] (x0, y0, width, height),
    in mpl's figure coordinates. However, the axes all share one space in
    plotly such that the domain will always be [0, 0, 1, 1]
    (x0, y0, x1, y1). To convert between the two, the mpl figure bounds
    need to be mapped to a [0, 1] domain for x and y. The margins set
    upon opening a new figure will appropriately match the mpl margins.

    Optionally, setting margins=0 and simply copying the domains from
    mpl to plotly would place axes appropriately. However,
    this would throw off axis and title labeling.

    Positional arguments:
    mpl_plot_bounds -- the (x0, y0, width, height) params for current ax **
    mpl_max_x_bounds -- overall (x0, x1) bounds for all axes **

    ** these are all specified in mpl figure coordinates

    """
    mpl_x_dom = [mpl_plot_bounds[0], mpl_plot_bounds[0] + mpl_plot_bounds[2]]
    plotting_width = mpl_max_x_bounds[1] - mpl_max_x_bounds[0]
    x0 = (mpl_x_dom[0] - mpl_max_x_bounds[0]) / plotting_width
    x1 = (mpl_x_dom[1] - mpl_max_x_bounds[0]) / plotting_width
    return [x0, x1]


def convert_y_domain(mpl_plot_bounds, mpl_max_y_bounds):
    """Map y dimension of current plot to plotly's domain space.

    The bbox used to locate an axes object in mpl differs from the
    method used to locate axes in plotly. The mpl version locates each
    axes in the figure so that axes in a single-plot figure might have
    the bounds, [0.125, 0.125, 0.775, 0.775] (x0, y0, width, height),
    in mpl's figure coordinates. However, the axes all share one space in
    plotly such that the domain will always be [0, 0, 1, 1]
    (x0, y0, x1, y1). To convert between the two, the mpl figure bounds
    need to be mapped to a [0, 1] domain for x and y. The margins set
    upon opening a new figure will appropriately match the mpl margins.

    Optionally, setting margins=0 and simply copying the domains from
    mpl to plotly would place axes appropriately. However,
    this would throw off axis and title labeling.

    Positional arguments:
    mpl_plot_bounds -- the (x0, y0, width, height) params for current ax **
    mpl_max_y_bounds -- overall (y0, y1) bounds for all axes **

    ** these are all specified in mpl figure coordinates

    """
    mpl_y_dom = [mpl_plot_bounds[1], mpl_plot_bounds[1] + mpl_plot_bounds[3]]
    plotting_height = mpl_max_y_bounds[1] - mpl_max_y_bounds[0]
    y0 = (mpl_y_dom[0] - mpl_max_y_bounds[0]) / plotting_height
    y1 = (mpl_y_dom[1] - mpl_max_y_bounds[0]) / plotting_height
    return [y0, y1]


def display_to_paper(x, y, layout):
    """Convert mpl display coordinates to plotly paper coordinates.

    Plotly references object positions with an (x, y) coordinate pair in either
    'data' or 'paper' coordinates which reference actual data in a plot or
    the entire plotly axes space where the bottom-left of the bottom-left
    plot has the location (x, y) = (0, 0) and the top-right of the top-right
    plot has the location (x, y) = (1, 1). Display coordinates in mpl reference
    objects with an (x, y) pair in pixel coordinates, where the bottom-left
    corner is at the location (x, y) = (0, 0) and the top-right corner is at
    the location (x, y) = (figwidth*dpi, figheight*dpi). Here, figwidth and
    figheight are in inches and dpi are the dots per inch resolution.

    """
    num_x = x - layout["margin"]["l"]
    den_x = layout["width"] - (layout["margin"]["l"] + layout["margin"]["r"])
    num_y = y - layout["margin"]["b"]
    den_y = layout["height"] - (layout["margin"]["b"] + layout["margin"]["t"])
    return num_x / den_x, num_y / den_y


def get_axes_bounds(fig):
    """Return the entire axes space for figure.

    An axes object in mpl is specified by its relation to the figure where
    (0,0) corresponds to the bottom-left part of the figure and (1,1)
    corresponds to the top-right. Margins exist in matplotlib because axes
    objects normally don't go to the edges of the figure.

    In plotly, the axes area (where all subplots go) is always specified with
    the domain [0,1] for both x and y. This function finds the smallest box,
    specified by two points, that all of the mpl axes objects fit into. This
    box is then used to map mpl axes domains to plotly axes domains.

    """
    x_min, x_max, y_min, y_max = [], [], [], []
    for axes_obj in fig.get_axes():
        bounds = axes_obj.get_position().bounds
        x_min.append(bounds[0])
        x_max.append(bounds[0] + bounds[2])
        y_min.append(bounds[1])
        y_max.append(bounds[1] + bounds[3])
    x_min, y_min, x_max, y_max = min(x_min), min(y_min), max(x_max), max(y_max)
    return (x_min, x_max), (y_min, y_max)


def get_axis_mirror(main_spine, mirror_spine):
    if main_spine and mirror_spine:
        return "ticks"
    elif main_spine and not mirror_spine:
        return False
    elif not main_spine and mirror_spine:
        return False  # can't handle this case yet!
    else:
        return False  # nuttin'!


def get_bar_gap(bar_starts, bar_ends, tol=1e-10):
    if len(bar_starts) == len(bar_ends) and len(bar_starts) > 1:
        sides1 = bar_starts[1:]
        sides2 = bar_ends[:-1]
        gaps = [s2 - s1 for s2, s1 in zip(sides1, sides2)]
        gap0 = gaps[0]
        uniform = all([abs(gap0 - gap) < tol for gap in gaps])
        if uniform:
            return gap0


def convert_rgba_array(color_list):
    clean_color_list = list()
    for c in color_list:
        clean_color_list += [
            (dict(r=int(c[0] * 255), g=int(c[1] * 255), b=int(c[2] * 255), a=c[3]))
        ]
    plotly_colors = list()
    for rgba in clean_color_list:
        plotly_colors += ["rgba({r},{g},{b},{a})".format(**rgba)]
    if len(plotly_colors) == 1:
        return plotly_colors[0]
    else:
        return plotly_colors


def convert_path_array(path_array):
    symbols = list()
    for path in path_array:
        symbols += [convert_path(path)]
    if len(symbols) == 1:
        return symbols[0]
    else:
        return symbols


def convert_linewidth_array(width_array):
    if len(width_array) == 1:
        return width_array[0]
    else:
        return width_array


def convert_size_array(size_array):
    size = [math.sqrt(s) for s in size_array]
    if len(size) == 1:
        return size[0]
    else:
        return size


def get_markerstyle_from_collection(props):
    markerstyle = dict(
        alpha=None,
        facecolor=convert_rgba_array(props["styles"]["facecolor"]),
        marker=convert_path_array(props["paths"]),
        edgewidth=convert_linewidth_array(props["styles"]["linewidth"]),
        # markersize=convert_size_array(props['styles']['size']),  # TODO!
        markersize=convert_size_array(props["mplobj"].get_sizes()),
        edgecolor=convert_rgba_array(props["styles"]["edgecolor"]),
    )
    return markerstyle


def get_rect_xmin(data):
    """Find minimum x value from four (x,y) vertices."""
    return min(data[0][0], data[1][0], data[2][0], data[3][0])


def get_rect_xmax(data):
    """Find maximum x value from four (x,y) vertices."""
    return max(data[0][0], data[1][0], data[2][0], data[3][0])


def get_rect_ymin(data):
    """Find minimum y value from four (x,y) vertices."""
    return min(data[0][1], data[1][1], data[2][1], data[3][1])


def get_rect_ymax(data):
    """Find maximum y value from four (x,y) vertices."""
    return max(data[0][1], data[1][1], data[2][1], data[3][1])


def get_spine_visible(ax, spine_key):
    """Return some spine parameters for the spine, `spine_key`."""
    spine = ax.spines[spine_key]
    ax_frame_on = ax.get_frame_on()
    if check_maplotlib_max_version('3', '1'):
        spine_frame_like = spine.is_frame_like() if hasattr(spine, 'is_frame_like') else True
    else:
        spine_frame_like = True
    if not spine.get_visible():
        return False
    elif not spine._edgecolor[-1]:  # user's may have set edgecolor alpha==0
        return False
    elif not ax_frame_on and spine_frame_like:
        return False
    elif ax_frame_on and spine_frame_like:
        return True
    elif not ax_frame_on and not spine_frame_like:
        return True  # we've already checked for that it's visible.
    else:
        return False  # oh man, and i thought we exhausted the options...


def is_bar(bar_containers, **props):
    """A test to decide whether a path is a bar from a vertical bar chart."""

    # is this patch in a bar container?
    for container in bar_containers:
        if props["mplobj"] in container:
            return True
    return False


def is_fancy_bbox(**props):
    """A test to decide whether a path is a simple FancyBboxPatch."""
    return FancyBboxPatch and isinstance(props.get("mplobj"), FancyBboxPatch)


def make_bar(**props):
    """Make an intermediate bar dictionary.

    This creates a bar dictionary which aids in the comparison of new bars to
    old bars from other bar chart (patch) collections. This is not the
    dictionary that needs to get passed to plotly as a data dictionary. That
    happens in PlotlyRenderer in that class's draw_bar method. In other
    words, this dictionary describes a SINGLE bar, whereas, plotly will
    require a set of bars to be passed in a data dictionary.

    """
    return {
        "bar": props["mplobj"],
        "x0": get_rect_xmin(props["data"]),
        "y0": get_rect_ymin(props["data"]),
        "x1": get_rect_xmax(props["data"]),
        "y1": get_rect_ymax(props["data"]),
        "alpha": props["style"]["alpha"],
        "edgecolor": props["style"]["edgecolor"],
        "facecolor": props["style"]["facecolor"],
        "edgewidth": props["style"]["edgewidth"],
        "dasharray": props["style"]["dasharray"],
        "zorder": props["style"]["zorder"],
    }


def prep_ticks(ax, index, ax_type, props):
    """Prepare axis obj belonging to axes obj.

    positional arguments:
    ax - the mpl axes instance
    index - the index of the axis in `props`
    ax_type - 'x' or 'y' (for now)
    props - an mplexporter poperties dictionary

    """
    axis_dict = dict()
    if ax_type == "x":
        axis = ax.get_xaxis()
    elif ax_type == "y":
        axis = ax.get_yaxis()
    else:
        return dict()  # whoops!

    scale = props["axes"][index]["scale"]
    if scale == "linear":
        # get tick location information
        try:
            tickvalues = props["axes"][index]["tickvalues"]
            tick0 = tickvalues[0]
            dticks = [
                round(tickvalues[i] - tickvalues[i - 1], 12)
                for i in range(1, len(tickvalues) - 1)
            ]
            if all([dticks[i] == dticks[i - 1] for i in range(1, len(dticks) - 1)]):
                dtick = tickvalues[1] - tickvalues[0]
            else:
                warnings.warn(
                    "'linear' {0}-axis tick spacing not even, "
                    "ignoring mpl tick formatting.".format(ax_type)
                )
                raise TypeError
        except (IndexError, TypeError):
            axis_dict["nticks"] = props["axes"][index]["nticks"]
        else:
            axis_dict["tick0"] = tick0
            axis_dict["dtick"] = dtick
            axis_dict["tickmode"] = None
        if ax_type == "x" and "xlim" in props:
            axis_dict["range"] = [props["xlim"][0], props["xlim"][1]]
            axis_dict["custom_range"] = True
        elif ax_type == "y" and "ylim" in props:
            axis_dict["range"] = [props["ylim"][0], props["ylim"][1]]
            axis_dict["custom_range"] = True
    elif scale == "log":
        try:
            axis_dict["tick0"] = props["axes"][index]["tickvalues"][0]
            axis_dict["dtick"] = (
                props["axes"][index]["tickvalues"][1]
                - props["axes"][index]["tickvalues"][0]
            )
            axis_dict["tickmode"] = None
        except (IndexError, TypeError):
            axis_dict = dict(nticks=props["axes"][index]["nticks"])
        base = axis.get_transform().base
        if base == 10:
            if ax_type == "x" and "xlim" in props:
                axis_dict["range"] = [
                    math.log10(props["xlim"][0]),
                    math.log10(props["xlim"][1]),
                ]
                axis_dict["custom_range"] = True
            elif ax_type == "y" and "ylim" in props:
                axis_dict["range"] = [
                    math.log10(props["ylim"][0]),
                    math.log10(props["ylim"][1]),
                ]
                axis_dict["custom_range"] = True
        else:
            axis_dict = dict(range=None, type="linear")
            warnings.warn(
                "Converted non-base10 {0}-axis log scale to 'linear'" "".format(ax_type)
            )
    else:
        return dict()
    # get tick label formatting information
    formatter = axis.get_major_formatter().__class__.__name__
    if ax_type == "x" and "DateFormatter" in formatter:
        axis_dict["type"] = "date"
        try:
            axis_dict["tick0"] = mpl_dates_to_datestrings(axis_dict["tick0"], formatter)
        except KeyError:
            pass
        finally:
            axis_dict.pop("dtick", None)
            axis_dict.pop("tickmode", None)
            axis_dict["range"] = mpl_dates_to_datestrings(props["xlim"], formatter)

    if formatter == "LogFormatterMathtext":
        axis_dict["exponentformat"] = "e"
    return axis_dict


def prep_xy_axis(ax, props, x_bounds, y_bounds):
    xaxis = dict(
        type=props["axes"][0]["scale"],
        range=list(props["xlim"]),
        showgrid=props["axes"][0]["grid"]["gridOn"],
        domain=convert_x_domain(props["bounds"], x_bounds),
        side=props["axes"][0]["position"],
        tickfont=dict(size=props["axes"][0]["fontsize"]),
    )
    xaxis.update(prep_ticks(ax, 0, "x", props))
    yaxis = dict(
        type=props["axes"][1]["scale"],
        range=list(props["ylim"]),
        showgrid=props["axes"][1]["grid"]["gridOn"],
        domain=convert_y_domain(props["bounds"], y_bounds),
        side=props["axes"][1]["position"],
        tickfont=dict(size=props["axes"][1]["fontsize"]),
    )
    yaxis.update(prep_ticks(ax, 1, "y", props))
    return xaxis, yaxis


def mpl_dates_to_datestrings(dates, mpl_formatter):
    """Convert matplotlib dates to iso-formatted-like time strings.

    Plotly's accepted format: "YYYY-MM-DD HH:MM:SS" (e.g., 2001-01-01 00:00:00)

    Info on mpl dates: http://matplotlib.org/api/dates_api.html

    """
    _dates = dates

    # this is a pandas datetime formatter, times show up in floating point days
    # since the epoch (1970-01-01T00:00:00+00:00)
    if mpl_formatter == "TimeSeries_DateFormatter":
        try:
            dates = matplotlib.dates.epoch2num([date * 24 * 60 * 60 for date in dates])
            dates = matplotlib.dates.num2date(dates)
        except:  # noqa: E722
            return _dates

    # the rest of mpl dates are in floating point days since
    # (0001-01-01T00:00:00+00:00) + 1. I.e., (0001-01-01T00:00:00+00:00) == 1.0
    # according to mpl --> try num2date(1)
    else:
        try:
            dates = matplotlib.dates.num2date(dates)
        except:  # noqa: E722
            return _dates

    time_stings = [
        " ".join(date.isoformat().split("+")[0].split("T")) for date in dates
    ]
    return time_stings


# dashed is dash in matplotlib
DASH_MAP = {
    "10,0": "solid",
    "6,6": "dash",
    "2,2": "circle",
    "4,4,2,4": "dashdot",
    "none": "solid",
    "7.4,3.2": "dash",
}

PATH_MAP = {
    ("M", "C", "C", "C", "C", "C", "C", "C", "C", "Z"): "o",
    ("M", "L", "L", "L", "L", "L", "L", "L", "L", "L", "Z"): "*",
    ("M", "L", "L", "L", "L", "L", "L", "L", "Z"): "8",
    ("M", "L", "L", "L", "L", "L", "Z"): "h",
    ("M", "L", "L", "L", "L", "Z"): "p",
    ("M", "L", "M", "L", "M", "L"): "1",
    ("M", "L", "L", "L", "Z"): "s",
    ("M", "L", "M", "L"): "+",
    ("M", "L", "L", "Z"): "^",
    ("M", "L"): "|",
}

SYMBOL_MAP = {
    "o": "circle",
    "v": "triangle-down",
    "^": "triangle-up",
    "<": "triangle-left",
    ">": "triangle-right",
    "s": "square",
    "+": "cross",
    "x": "x",
    "*": "star",
    "D": "diamond",
    "d": "diamond",
}

VA_MAP = {"center": "middle", "baseline": "bottom", "top": "top"}


# clearml/utilities/plotlympl/renderer.py

"""
Renderer Module

This module defines the PlotlyRenderer class and a single function,
fig_to_plotly, which is intended to be the main way that user's will interact
with the matplotlylib package.

"""
from __future__ import absolute_import

import six
import warnings

from .mplexporter import Renderer
from . import mpltools


# Warning format
def warning_on_one_line(msg, category, filename, lineno, file=None, line=None):
    return "%s:%s: %s:\n\n%s\n\n" % (filename, lineno, category.__name__, msg)


warnings.formatwarning = warning_on_one_line


class PlotlyRenderer(Renderer):
    """A renderer class inheriting from base for rendering mpl plots in plotly.

    A renderer class to be used with an exporter for rendering matplotlib
    plots in Plotly. This module defines the PlotlyRenderer class which handles
    the creation of the JSON structures that get sent to plotly.

    All class attributes available are defined in __init__().

    Basic Usage:

    # (mpl code) #
    fig = gcf()
    renderer = PlotlyRenderer(fig)
    exporter = Exporter(renderer)
    exporter.run(fig)  # ... et voila

    """

    def __init__(self):
        """Initialize PlotlyRenderer obj.

        PlotlyRenderer obj is called on by an Exporter object to draw
        matplotlib objects like figures, axes, text, etc.

        All class attributes are listed here in the __init__ method.

        """
        self.plotly_fig = dict(data=[], layout={})
        self.mpl_fig = None
        self.current_mpl_ax = None
        self.bar_containers = None
        self.current_bars = []
        self.current_bars_names = []
        self.axis_ct = 0
        self.x_is_mpl_date = False
        self.mpl_x_bounds = (0, 1)
        self.mpl_y_bounds = (0, 1)
        self.msg = "Initialized PlotlyRenderer\n"

    def open_figure(self, fig, props):
        """Creates a new figure by beginning to fill out layout dict.

        The 'autosize' key is set to false so that the figure will mirror
        sizes set by mpl. The 'hovermode' key controls what shows up when you
        mouse around a figure in plotly, it's set to show the 'closest' point.

        Positional agurments:
        fig -- a matplotlib.figure.Figure object.
        props.keys(): [
            'figwidth',
            'figheight',
            'dpi'
            ]

        """
        self.msg += "Opening figure\n"
        self.mpl_fig = fig
        self.plotly_fig["layout"] = dict(
            width=int(props["figwidth"] * props["dpi"]),
            height=int(props["figheight"] * props["dpi"]),
            autosize=False,
            hovermode="closest",
        )
        self.mpl_x_bounds, self.mpl_y_bounds = mpltools.get_axes_bounds(fig)
        margin = dict(
            l=int(self.mpl_x_bounds[0] * self.plotly_fig["layout"]["width"]),
            r=int((1 - self.mpl_x_bounds[1]) * self.plotly_fig["layout"]["width"]),
            t=int((1 - self.mpl_y_bounds[1]) * self.plotly_fig["layout"]["height"]),
            b=int(self.mpl_y_bounds[0] * self.plotly_fig["layout"]["height"]),
            pad=0,
        )
        self.plotly_fig["layout"]["margin"] = margin

    def close_figure(self, fig):
        """Closes figure by cleaning up data and layout dictionaries.

        The PlotlyRenderer's job is to create an appropriate set of data and
        layout dictionaries. When the figure is closed, some cleanup and
        repair is necessary. This method removes inappropriate dictionary
        entries, freeing up Plotly to use defaults and best judgements to
        complete the entries. This method is called by an Exporter object.

        Positional arguments:
        fig -- a matplotlib.figure.Figure object.

        """
        self.plotly_fig["layout"]["showlegend"] = False
        self.msg += "Closing figure\n"

    def open_axes(self, ax, props):
        """Setup a new axes object (subplot in plotly).

        Plotly stores information about subplots in different 'xaxis' and
        'yaxis' objects which are numbered. These are just dictionaries
        included in the layout dictionary. This function takes information
        from the Exporter, fills in appropriate dictionary entries,
        and updates the layout dictionary. PlotlyRenderer keeps track of the
        number of plots by incrementing the axis_ct attribute.

        Setting the proper plot domain in plotly is a bit tricky. Refer to
        the documentation for mpltools.convert_x_domain and
        mpltools.convert_y_domain.

        Positional arguments:
        ax -- an mpl axes object. This will become a subplot in plotly.
        props.keys() -- [
            'axesbg',           (background color for axes obj)
            'axesbgalpha',      (alpha, or opacity for background)
            'bounds',           ((x0, y0, width, height) for axes)
            'dynamic',          (zoom/pan-able?)
            'axes',             (list: [xaxis, yaxis])
            'xscale',           (log, linear, or date)
            'yscale',
            'xlim',             (range limits for x)
            'ylim',
            'xdomain'           (xdomain=xlim, unless it's a date)
            'ydomain'
            ]

        """
        self.msg += "  Opening axes\n"
        self.current_mpl_ax = ax
        self.bar_containers = [
            c
            for c in ax.containers  # empty is OK
            if c.__class__.__name__ == "BarContainer"
        ]
        self.current_bars = []

        # set defaults in axes
        xaxis = dict(
            anchor="y{0}".format(self.axis_ct or ''), zeroline=False, ticks="inside"
        )
        yaxis = dict(
            anchor="x{0}".format(self.axis_ct or ''), zeroline=False, ticks="inside"
        )
        # update defaults with things set in mpl
        mpl_xaxis, mpl_yaxis = mpltools.prep_xy_axis(
            ax=ax, props=props, x_bounds=self.mpl_x_bounds, y_bounds=self.mpl_y_bounds
        )
        xaxis.update(mpl_xaxis)
        yaxis.update(mpl_yaxis)
        bottom_spine = mpltools.get_spine_visible(ax, "bottom")
        top_spine = mpltools.get_spine_visible(ax, "top")
        left_spine = mpltools.get_spine_visible(ax, "left")
        right_spine = mpltools.get_spine_visible(ax, "right")
        xaxis["mirror"] = mpltools.get_axis_mirror(bottom_spine, top_spine)
        yaxis["mirror"] = mpltools.get_axis_mirror(left_spine, right_spine)
        xaxis["showline"] = bottom_spine
        yaxis["showline"] = top_spine

        # put axes in our figure
        self.plotly_fig["layout"]["xaxis{0}".format(self.axis_ct or '')] = xaxis
        self.plotly_fig["layout"]["yaxis{0}".format(self.axis_ct or '')] = yaxis

        # let all subsequent dates be handled properly if required

        if xaxis.get("type") == "date":
            self.x_is_mpl_date = True

        self.axis_ct += 1

    def close_axes(self, ax):
        """Close the axes object and clean up.

        Bars from bar charts are given to PlotlyRenderer one-by-one,
        thus they need to be taken care of at the close of each axes object.
        The self.current_bars variable should be empty unless a bar
        chart has been created.

        Positional arguments:
        ax -- an mpl axes object, not required at this time.

        """
        if self.current_bars:
            # noinspection PyBroadException
            try:
                self.current_bars_names = [n.get_text() for n in ax.legend().texts]
            except Exception:
                pass
        self.draw_bars(self.current_bars)
        self.msg += "  Closing axes\n"
        self.x_is_mpl_date = False

    def draw_bars(self, bars):

        # sort bars according to bar containers
        mpl_traces = []
        for container in self.bar_containers:
            mpl_traces.append(
                [
                    bar_props
                    for bar_props in self.current_bars
                    if bar_props["mplobj"] in container
                ]
            )
        for i, trace in enumerate(mpl_traces):
            self.draw_bar(trace, self.current_bars_names[i] if i < len(self.current_bars_names) else None)

    def draw_bar(self, coll, name=None):
        """Draw a collection of similar patches as a bar chart.

        After bars are sorted, an appropriate data dictionary must be created
        to tell plotly about this data. Just like draw_line or draw_markers,
        draw_bar translates patch/path information into something plotly
        understands.

        Positional arguments:
        patch_coll -- a collection of patches to be drawn as a bar chart.

        """
        tol = 1e-10
        trace = [mpltools.make_bar(**bar_props) for bar_props in coll]
        widths = [bar_props["x1"] - bar_props["x0"] for bar_props in trace]
        heights = [bar_props["y1"] - bar_props["y0"] for bar_props in trace]
        vertical = abs(sum(widths[0] - widths[iii] for iii in range(len(widths)))) < tol
        horizontal = (
            abs(sum(heights[0] - heights[iii] for iii in range(len(heights)))) < tol
        )
        if (vertical and horizontal) or (not vertical and not horizontal):
            # Check for monotonic x. Can't both be true!
            x_zeros = [bar_props["x0"] for bar_props in trace]
            if all(
                (x_zeros[iii + 1] > x_zeros[iii] for iii in range(len(x_zeros[:-1])))
            ):
                orientation = "v"
            else:
                orientation = "h"
        elif vertical:
            orientation = "v"
        else:
            orientation = "h"
        if orientation == "v":
            self.msg += "    Attempting to draw a vertical bar chart\n"
            old_heights = [bar_props["y1"] for bar_props in trace]
            for bar in trace:
                bar["y0"], bar["y1"] = 0, bar["y1"] - bar["y0"]
            new_heights = [bar_props["y1"] for bar_props in trace]
            # check if we're stacked or not...
            for old, new in zip(old_heights, new_heights):
                if abs(old - new) > tol:
                    self.plotly_fig["layout"]["barmode"] = "stack"
                    self.plotly_fig["layout"]["hovermode"] = "x"
            x = [bar["x0"] + (bar["x1"] - bar["x0"]) / 2 for bar in trace]
            y = [bar["y1"] for bar in trace]
            bar_gap = mpltools.get_bar_gap(
                [bar["x0"] for bar in trace], [bar["x1"] for bar in trace]
            )
            if self.x_is_mpl_date:
                x = [bar["x0"] for bar in trace]
                formatter = (
                    self.current_mpl_ax.get_xaxis()
                    .get_major_formatter()
                    .__class__.__name__
                )
                x = mpltools.mpl_dates_to_datestrings(x, formatter)
        else:
            self.msg += "    Attempting to draw a horizontal bar chart\n"
            old_rights = [bar_props["x1"] for bar_props in trace]
            for bar in trace:
                bar["x0"], bar["x1"] = 0, bar["x1"] - bar["x0"]
            new_rights = [bar_props["x1"] for bar_props in trace]
            # check if we're stacked or not...
            for old, new in zip(old_rights, new_rights):
                if abs(old - new) > tol:
                    self.plotly_fig["layout"]["barmode"] = "stack"
                    self.plotly_fig["layout"]["hovermode"] = "y"
            x = [bar["x1"] for bar in trace]
            y = [bar["y0"] + (bar["y1"] - bar["y0"]) / 2 for bar in trace]
            bar_gap = mpltools.get_bar_gap(
                [bar["y0"] for bar in trace], [bar["y1"] for bar in trace]
            )
        bar = dict(
            type="bar",
            orientation=orientation,
            x=x,
            y=y,
            xaxis="x{0}".format(self.axis_ct),
            yaxis="y{0}".format(self.axis_ct),
            opacity=trace[0]["alpha"],  # TODO: get all alphas if array?
            marker=dict(
                color=trace[0]["facecolor"],  # TODO: get all
                line=dict(width=trace[0]["edgewidth"]),
            ),
        )  # TODO ditto
        if name:
            bar["name"] = name
        if len(bar["x"]) > 1:
            self.msg += "    Heck yeah, I drew that bar chart\n"
            self.plotly_fig['data'].append(bar)
            if bar_gap is not None:
                self.plotly_fig["layout"]["bargap"] = bar_gap
        else:
            self.msg += "    Bar chart not drawn\n"
            warnings.warn(
                "found box chart data with length <= 1, "
                "assuming data redundancy, not plotting."
            )

    def draw_marked_line(self, **props):
        """Create a data dict for a line obj.

        This will draw 'lines', 'markers', or 'lines+markers'.

        props.keys() -- [
        'coordinates',  ('data', 'axes', 'figure', or 'display')
        'data',         (a list of xy pairs)
        'mplobj',       (the matplotlib.lines.Line2D obj being rendered)
        'label',        (the name of the Line2D obj being rendered)
        'linestyle',    (linestyle dict, can be None, see below)
        'markerstyle',  (markerstyle dict, can be None, see below)
        ]

        props['linestyle'].keys() -- [
        'alpha',        (opacity of Line2D obj)
        'color',        (color of the line if it exists, not the marker)
        'linewidth',
        'dasharray',    (code for linestyle, see DASH_MAP in mpltools.py)
        'zorder',       (viewing precedence when stacked with other objects)
        ]

        props['markerstyle'].keys() -- [
        'alpha',        (opacity of Line2D obj)
        'marker',       (the mpl marker symbol, see SYMBOL_MAP in mpltools.py)
        'facecolor',    (color of the marker face)
        'edgecolor',    (color of the marker edge)
        'edgewidth',    (width of marker edge)
        'markerpath',   (an SVG path for drawing the specified marker)
        'zorder',       (viewing precedence when stacked with other objects)
        ]

        """
        self.msg += "    Attempting to draw a line "
        line, marker = {}, {}
        if props["linestyle"] and props["markerstyle"]:
            self.msg += "... with both lines+markers\n"
            mode = "lines+markers"
        elif props["linestyle"]:
            self.msg += "... with just lines\n"
            mode = "lines"
        elif props["markerstyle"]:
            self.msg += "... with just markers\n"
            mode = "markers"
        if props["linestyle"]:
            color = mpltools.merge_color_and_opacity(
                props["linestyle"]["color"], props["linestyle"]["alpha"]
            )

            # print(mpltools.convert_dash(props['linestyle']['dasharray']))
            line = dict(
                color=color,
                width=props["linestyle"]["linewidth"],
                dash=mpltools.convert_dash(props["linestyle"]["dasharray"]),
            )
        if props["markerstyle"]:
            marker = dict(
                opacity=props["markerstyle"]["alpha"],
                color=props["markerstyle"]["facecolor"],
                symbol=mpltools.convert_symbol(props["markerstyle"]["marker"]),
                size=props["markerstyle"]["markersize"],
                line=dict(
                    color=props["markerstyle"]["edgecolor"],
                    width=props["markerstyle"]["edgewidth"],
                ),
            )
        if props["coordinates"] == "data":
            marked_line = dict(
                type="scatter",
                mode=mode,
                name=(
                    str(props["label"])
                    if isinstance(props["label"], six.string_types)
                    else props["label"]
                ),
                x=[xy_pair[0] for xy_pair in props["data"]],
                y=[xy_pair[1] for xy_pair in props["data"]],
                xaxis="x{0}".format(self.axis_ct),
                yaxis="y{0}".format(self.axis_ct),
                line=line,
                marker=marker,
            )
            if self.x_is_mpl_date:
                formatter = (
                    self.current_mpl_ax.get_xaxis()
                    .get_major_formatter()
                    .__class__.__name__
                )
                marked_line["x"] = mpltools.mpl_dates_to_datestrings(
                    marked_line["x"], formatter
                )
            self.plotly_fig['data'].append(marked_line)
            self.msg += "    Heck yeah, I drew that line\n"
        else:
            self.msg += "    Line didn't have 'data' coordinates, " "not drawing\n"
            warnings.warn(
                "Bummer! Plotly can currently only draw Line2D "
                "objects from matplotlib that are in 'data' "
                "coordinates!"
            )

    def draw_image(self, **props):
        """Draw image.

        Not implemented yet!

        """
        self.msg += "    Attempting to draw image\n"
        self.msg += "    Not drawing image\n"
        warnings.warn(
            "Aw. Snap! You're gonna have to hold off on "
            "the selfies for now. Plotly can't import "
            "images from matplotlib yet!"
        )

    def draw_path_collection(self, **props):
        """Add a path collection to data list as a scatter plot.

        Current implementation defaults such collections as scatter plots.
        Matplotlib supports collections that have many of the same parameters
        in common like color, size, path, etc. However, they needn't all be
        the same. Plotly does not currently support such functionality and
        therefore, the style for the first object is taken and used to define
        the remaining paths in the collection.

        props.keys() -- [
        'paths',                (structure: [vertices, path_code])
        'path_coordinates',     ('data', 'axes', 'figure', or 'display')
        'path_transforms',      (mpl transform, including Affine2D matrix)
        'offsets',              (offset from axes, helpful if in 'data')
        'offset_coordinates',   ('data', 'axes', 'figure', or 'display')
        'offset_order',
        'styles',               (style dict, see below)
        'mplobj'                (the collection obj being drawn)
        ]

        props['styles'].keys() -- [
        'linewidth',            (one or more linewidths)
        'facecolor',            (one or more facecolors for path)
        'edgecolor',            (one or more edgecolors for path)
        'alpha',                (one or more opacites for path)
        'zorder',               (precedence when stacked)
        ]

        """
        self.msg += "    Attempting to draw a path collection\n"
        if props["offset_coordinates"] == "data":
            markerstyle = mpltools.get_markerstyle_from_collection(props)
            scatter_props = {
                "coordinates": "data",
                "data": props["offsets"],
                "label": None,
                "markerstyle": markerstyle,
                "linestyle": None,
            }
            self.msg += "    Drawing path collection as markers\n"
            self.draw_marked_line(**scatter_props)
        else:
            self.msg += "    Path collection not linked to 'data', " "not drawing\n"
            warnings.warn(
                "Dang! That path collection is out of this "
                "world. I totally don't know what to do with "
                "it yet! Plotly can only import path "
                "collections linked to 'data' coordinates"
            )

    def draw_path(self, **props):
        """Draw path, currently only attempts to draw bar charts.

        This function attempts to sort a given path into a collection of
        horizontal or vertical bar charts. Most of the actual code takes
        place in functions from mpltools.py.

        props.keys() -- [
        'data',         (a list of verticies for the path)
        'coordinates',  ('data', 'axes', 'figure', or 'display')
        'pathcodes',    (code for the path, structure: ['M', 'L', 'Z', etc.])
        'style',        (style dict, see below)
        'mplobj'        (the mpl path object)
        ]

        props['style'].keys() -- [
        'alpha',        (opacity of path obj)
        'edgecolor',
        'facecolor',
        'edgewidth',
        'dasharray',    (style for path's enclosing line)
        'zorder'        (precedence of obj when stacked)
        ]

        """
        self.msg += "    Attempting to draw a path\n"
        is_bar = mpltools.is_bar(self.current_mpl_ax.containers, **props)
        if is_bar:
            self.current_bars += [props]
        elif mpltools.is_fancy_bbox(**props):
            self.current_bars += [props]
        else:
            self.msg += "    This path isn't a bar, not drawing\n"
            warnings.warn(
                "I found a path object that I don't think is part "
                "of a bar chart. Ignoring."
            )

    def draw_text(self, **props):
        """Create an annotation dict for a text obj.

        Currently, plotly uses either 'page' or 'data' to reference
        annotation locations. These refer to 'display' and 'data',
        respectively for the 'coordinates' key used in the Exporter.
        Appropriate measures are taken to transform text locations to
        reference one of these two options.

        props.keys() -- [
        'text',         (actual content string, not the text obj)
        'position',     (an x, y pair, not an mpl Bbox)
        'coordinates',  ('data', 'axes', 'figure', 'display')
        'text_type',    ('title', 'xlabel', or 'ylabel')
        'style',        (style dict, see below)
        'mplobj'        (actual mpl text object)
        ]

        props['style'].keys() -- [
        'alpha',        (opacity of text)
        'fontsize',     (size in points of text)
        'color',        (hex color)
        'halign',       (horizontal alignment, 'left', 'center', or 'right')
        'valign',       (vertical alignment, 'baseline', 'center', or 'top')
        'rotation',
        'zorder',       (precedence of text when stacked with other objs)
        ]

        """
        self.msg += "    Attempting to draw an mpl text object\n"
        if not mpltools.check_corners(props["mplobj"], self.mpl_fig):
            warnings.warn(
                "Looks like the annotation(s) you are trying \n"
                "to draw lies/lay outside the given figure size.\n\n"
                "Therefore, the resulting Plotly figure may not be \n"
                "large enough to view the full text. To adjust \n"
                "the size of the figure, use the 'width' and \n"
                "'height' keys in the Layout object. Alternatively,\n"
                "use the Margin object to adjust the figure's margins."
            )
        align = props["mplobj"]._multialignment
        if not align:
            align = props["style"]["halign"]  # mpl default
        if "annotations" not in self.plotly_fig["layout"]:
            self.plotly_fig["layout"]["annotations"] = []
        if props["text_type"] == "xlabel":
            self.msg += "      Text object is an xlabel\n"
            self.draw_xlabel(**props)
        elif props["text_type"] == "ylabel":
            self.msg += "      Text object is a ylabel\n"
            self.draw_ylabel(**props)
        elif props["text_type"] == "title":
            self.msg += "      Text object is a title\n"
            self.draw_title(**props)
        else:  # just a regular text annotation...
            self.msg += "      Text object is a normal annotation\n"
            if props["coordinates"] != "data":
                self.msg += (
                    "        Text object isn't linked to 'data' " "coordinates\n"
                )
                x_px, y_px = (
                    props["mplobj"].get_transform().transform(props["position"])
                )
                x, y = mpltools.display_to_paper(x_px, y_px, self.plotly_fig["layout"])
                xref = "paper"
                yref = "paper"
                xanchor = props["style"]["halign"]  # no difference here!
                yanchor = mpltools.convert_va(props["style"]["valign"])
            else:
                self.msg += "        Text object is linked to 'data' " "coordinates\n"
                x, y = props["position"]
                axis_ct = self.axis_ct
                xaxis = self.plotly_fig["layout"]["xaxis{0}".format(axis_ct or '')]
                yaxis = self.plotly_fig["layout"]["yaxis{0}".format(axis_ct or '')]
                if (
                    xaxis["range"][0] < x < xaxis["range"][1]
                    and yaxis["range"][0] < y < yaxis["range"][1]
                ):
                    xref = "x{0}".format(self.axis_ct)
                    yref = "y{0}".format(self.axis_ct)
                else:
                    self.msg += (
                        "            Text object is outside "
                        "plotting area, making 'paper' reference.\n"
                    )
                    x_px, y_px = (
                        props["mplobj"].get_transform().transform(props["position"])
                    )
                    x, y = mpltools.display_to_paper(
                        x_px, y_px, self.plotly_fig["layout"]
                    )
                    xref = "paper"
                    yref = "paper"
                xanchor = props["style"]["halign"]  # no difference here!
                yanchor = mpltools.convert_va(props["style"]["valign"])
            annotation = dict(
                text=(
                    str(props["text"])
                    if isinstance(props["text"], six.string_types)
                    else props["text"]
                ),
                opacity=props["style"]["alpha"],
                x=x,
                y=y,
                xref=xref,
                yref=yref,
                align=align,
                xanchor=xanchor,
                yanchor=yanchor,
                showarrow=False,  # change this later?
                font=dict(
                    color=props["style"]["color"], size=props["style"]["fontsize"]
                ),
            )
            self.plotly_fig["layout"]["annotations"] += (annotation,)
            self.msg += "    Heck, yeah I drew that annotation\n"

    def draw_title(self, **props):
        """Add a title to the current subplot in layout dictionary.

        If there exists more than a single plot in the figure, titles revert
        to 'page'-referenced annotations.

        props.keys() -- [
        'text',         (actual content string, not the text obj)
        'position',     (an x, y pair, not an mpl Bbox)
        'coordinates',  ('data', 'axes', 'figure', 'display')
        'text_type',    ('title', 'xlabel', or 'ylabel')
        'style',        (style dict, see below)
        'mplobj'        (actual mpl text object)
        ]

        props['style'].keys() -- [
        'alpha',        (opacity of text)
        'fontsize',     (size in points of text)
        'color',        (hex color)
        'halign',       (horizontal alignment, 'left', 'center', or 'right')
        'valign',       (vertical alignment, 'baseline', 'center', or 'top')
        'rotation',
        'zorder',       (precedence of text when stacked with other objs)
        ]

        """
        self.msg += "        Attempting to draw a title\n"
        if len(self.mpl_fig.axes) > 1:
            self.msg += (
                "          More than one subplot, adding title as " "annotation\n"
            )
            x_px, y_px = props["mplobj"].get_transform().transform(props["position"])
            x, y = mpltools.display_to_paper(x_px, y_px, self.plotly_fig["layout"])
            annotation = dict(
                text=props["text"],
                font=dict(
                    color=props["style"]["color"], size=props["style"]["fontsize"]
                ),
                xref="paper",
                yref="paper",
                x=x,
                y=y,
                xanchor="center",
                yanchor="bottom",
                showarrow=False,  # no arrow for a title!
            )
            self.plotly_fig["layout"]["annotations"] += (annotation,)
        else:
            self.msg += (
                "          Only one subplot found, adding as a " "plotly title\n"
            )
            self.plotly_fig["layout"]["title"] = props["text"]
            titlefont = dict(
                size=props["style"]["fontsize"], color=props["style"]["color"]
            )
            self.plotly_fig["layout"]["titlefont"] = titlefont

    def draw_xlabel(self, **props):
        """Add an xaxis label to the current subplot in layout dictionary.

        props.keys() -- [
        'text',         (actual content string, not the text obj)
        'position',     (an x, y pair, not an mpl Bbox)
        'coordinates',  ('data', 'axes', 'figure', 'display')
        'text_type',    ('title', 'xlabel', or 'ylabel')
        'style',        (style dict, see below)
        'mplobj'        (actual mpl text object)
        ]

        props['style'].keys() -- [
        'alpha',        (opacity of text)
        'fontsize',     (size in points of text)
        'color',        (hex color)
        'halign',       (horizontal alignment, 'left', 'center', or 'right')
        'valign',       (vertical alignment, 'baseline', 'center', or 'top')
        'rotation',
        'zorder',       (precedence of text when stacked with other objs)
        ]

        """
        self.msg += "        Adding xlabel\n"
        axis_key = "xaxis{0}".format(self.axis_ct or '')
        # bugfix: add on last axis, self.axis_ct-1
        if axis_key not in self.plotly_fig["layout"]:
            axis_key = "xaxis{0}".format(max(0, self.axis_ct - 1) or '')
        self.plotly_fig["layout"][axis_key]["title"] = str(props["text"])
        titlefont = dict(size=props["style"]["fontsize"], color=props["style"]["color"])
        self.plotly_fig["layout"][axis_key]["titlefont"] = titlefont

    def draw_ylabel(self, **props):
        """Add a yaxis label to the current subplot in layout dictionary.

        props.keys() -- [
        'text',         (actual content string, not the text obj)
        'position',     (an x, y pair, not an mpl Bbox)
        'coordinates',  ('data', 'axes', 'figure', 'display')
        'text_type',    ('title', 'xlabel', or 'ylabel')
        'style',        (style dict, see below)
        'mplobj'        (actual mpl text object)
        ]

        props['style'].keys() -- [
        'alpha',        (opacity of text)
        'fontsize',     (size in points of text)
        'color',        (hex color)
        'halign',       (horizontal alignment, 'left', 'center', or 'right')
        'valign',       (vertical alignment, 'baseline', 'center', or 'top')
        'rotation',
        'zorder',       (precedence of text when stacked with other objs)
        ]

        """
        self.msg += "        Adding ylabel\n"
        axis_key = "yaxis{0}".format(self.axis_ct or '')
        # bugfix: add on last axis, self.axis_ct-1
        if axis_key not in self.plotly_fig["layout"]:
            axis_key = "yaxis{0}".format(max(0, self.axis_ct - 1) or '')
        self.plotly_fig["layout"][axis_key]["title"] = props["text"]
        titlefont = dict(size=props["style"]["fontsize"], color=props["style"]["color"])
        self.plotly_fig["layout"][axis_key]["titlefont"] = titlefont

    def resize(self):
        """Revert figure layout to allow plotly to resize.

        By default, PlotlyRenderer tries its hardest to precisely mimic an
        mpl figure. However, plotly is pretty good with aesthetics. By
        running PlotlyRenderer.resize(), layout parameters are deleted. This
        lets plotly choose them instead of mpl.

        """
        self.msg += "Resizing figure, deleting keys from layout\n"
        for key in ["width", "height", "autosize", "margin"]:
            try:
                del self.plotly_fig["layout"][key]
            except (KeyError, AttributeError):
                pass

    def strip_style(self):
        self.msg += "Stripping mpl style is no longer supported\n"


# clearml/utilities/process/mp.py

import os
import pickle
import struct
import sys
from functools import partial
from multiprocessing import Process, Semaphore, Event as ProcessEvent
from threading import Thread, Event as TrEvent, RLock as ThreadRLock
from time import sleep, time
from typing import List, Dict, Optional

import psutil
from six.moves.queue import Empty, Queue as TrQueue

from ..py3_interop import AbstractContextManager

try:
    from multiprocessing import SimpleQueue
except ImportError:
    from multiprocessing.queues import SimpleQueue

# Windows/MacOS compatibility
try:
    from multiprocessing.context import ForkContext  # noqa
except ImportError:
    ForkContext = None

# PY2 compatibility
try:
    from multiprocessing import get_context
except ImportError:
    def get_context(*args, **kwargs):  # noqa
        import multiprocessing
        return multiprocessing


class _ForkSafeThreadSyncObject(object):
    __process_lock = get_context("fork" if sys.platform == "linux" else "spawn").Lock()

    @classmethod
    def _inner_lock(cls):
        try:
            # let's agree that 90sec should be enough to get a lock on such a short protected piece of code.
            # if we failed, we have to assume some deadlock happened
            # (Python is not very safe with regrades to Process Locks)
            cls.__process_lock.acquire(block=True, timeout=90)
        except:  # noqa
            pass

    @classmethod
    def _inner_unlock(cls):
        try:
            cls.__process_lock.release()
        except:  # noqa
            # if we fail to release we might not have locked it in the first place (see timeout)
            pass

    def __init__(self, functor):
        self._sync = None
        self._instance_pid = None
        self._functor = functor

    def _create(self):
        # this part is not atomic, and there is not a lot we can do about it.
        if self._instance_pid != os.getpid() or not self._sync:
            # Notice! This is NOT atomic, this means the first time accessed, two concurrent calls might
            # end up overwriting each others, object
            # even tough it sounds horrible, the worst case in our usage scenario
            # is the first call usage is not "atomic"

            # Notice the order! we first create the object and THEN update the pid,
            # this is so whatever happens we Never try to used the old (pre-forked copy) of the synchronization object
            try:
                self._inner_lock()

                # we have to check gain inside the protected locked area
                if self._instance_pid != os.getpid() or not self._sync:
                    self._sync = self._functor()
                    self._instance_pid = os.getpid()
            finally:
                self._inner_unlock()


class ForkSafeRLock(_ForkSafeThreadSyncObject):
    def __init__(self):
        super(ForkSafeRLock, self).__init__(ThreadRLock)

    def acquire(self, *args, **kwargs):
        self._create()
        return self._sync.acquire(*args, **kwargs)

    def release(self, *args, **kwargs):
        if self._sync is None:
            return None
        self._create()
        return self._sync.release(*args, **kwargs)

    def __enter__(self):
        """Return `self` upon entering the runtime context."""
        self.acquire()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        """Raise any exception triggered within the runtime context."""
        # Do whatever cleanup.
        self.release()

    def _is_owned(self):
        self._create()
        return self._sync._is_owned()  # noqa


class ForkSemaphore(_ForkSafeThreadSyncObject):
    def __init__(self, value=1):
        super(ForkSemaphore, self).__init__(functor=partial(Semaphore, value))

    def acquire(self, *args, **kwargs):
        try:
            self._create()
        except BaseException:  # noqa
            return None

        return self._sync.acquire(*args, **kwargs)

    def release(self, *args, **kwargs):
        if self._sync is None:
            return None
        self._create()
        return self._sync.release(*args, **kwargs)

    def get_value(self):
        self._create()
        return self._sync.get_value()

    def __enter__(self):
        """Return `self` upon entering the runtime context."""
        self.acquire()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        """Raise any exception triggered within the runtime context."""
        # Do whatever cleanup.
        self.release()


class ForkEvent(_ForkSafeThreadSyncObject):
    def __init__(self):
        super(ForkEvent, self).__init__(TrEvent)

    def set(self):
        self._create()
        return self._sync.set()

    def clear(self):
        if self._sync is None:
            return None
        self._create()
        return self._sync.clear()

    def is_set(self):
        self._create()
        return self._sync.is_set()

    def wait(self, *args, **kwargs):
        self._create()
        return self._sync.wait(*args, **kwargs)


class ForkQueue(_ForkSafeThreadSyncObject):
    def __init__(self):
        super(ForkQueue, self).__init__(TrQueue)

    def get(self, *args, **kwargs):
        self._create()
        return self._sync.get(*args, **kwargs)

    def put(self, *args, **kwargs):
        self._create()
        return self._sync.put(*args, **kwargs)

    def empty(self):
        if not self._sync:
            return True
        self._create()
        return self._sync.empty()

    def full(self):
        if not self._sync:
            return False
        self._create()
        return self._sync.full()

    def close(self):
        if not self._sync:
            return
        self._create()
        return self._sync.close()


class ThreadCalls(object):
    def __init__(self):
        self._queue = ForkQueue()
        self._thread = Thread(target=self._worker)
        self._thread.daemon = True
        self._thread.start()

    def is_alive(self):
        return bool(self._thread) and self._thread.is_alive()

    def apply_async(self, func, args=None):
        if not func:
            return False
        self._queue.put((func, args))
        return True

    def close(self, timeout=5.):
        t = self._thread
        if not t:
            return
        try:
            # push something into queue so it knows this is the end
            self._queue.put(None)
            # wait fot thread it should not take long, so we have a 5 second timeout
            # the background thread itself is doing nothing but push into a queue, so it should not take long
            t.join(timeout=timeout)
        except BaseException:  # noqa
            pass
        # mark thread is done
        self._thread = None

    def _worker(self):
        while True:
            try:
                request = self._queue.get(block=True, timeout=1.0)
                if not request:
                    break
            except Empty:
                continue
            # noinspection PyBroadException
            try:
                if request[1] is not None:
                    request[0](*request[1])
                else:
                    request[0]()
            except Exception:
                pass
        self._thread = None


class SingletonThreadPool(object):
    __thread_pool = None
    __thread_pool_pid = None

    @classmethod
    def get(cls):
        if os.getpid() != cls.__thread_pool_pid:
            cls.__thread_pool = ThreadCalls()
            cls.__thread_pool_pid = os.getpid()
        return cls.__thread_pool

    @classmethod
    def clear(cls):
        if cls.__thread_pool:
            cls.__thread_pool.close()
        cls.__thread_pool = None
        cls.__thread_pool_pid = None

    @classmethod
    def is_active(cls):
        return cls.__thread_pool and cls.__thread_pool_pid == os.getpid() and cls.__thread_pool.is_alive()


class SafeQueue(object):
    """
    Many writers Single Reader multiprocessing safe Queue
    """
    __thread_pool = SingletonThreadPool()

    def __init__(self, *args, **kwargs):
        self._reader_thread = None
        self._reader_thread_started = False
        # Fix the python Queue and Use SimpleQueue write so it uses a single OS write,
        # making it atomic message passing
        self._q = SimpleQueue(*args, **kwargs)
        # noinspection PyBroadException
        try:
            # noinspection PyUnresolvedReferences,PyProtectedMember
            self._q._writer._send_bytes = partial(SafeQueue._pipe_override_send_bytes, self._q._writer)
        except Exception:
            pass
        self._internal_q = None
        # Note we should Never! assign a new object to `self._q_size`, just work with the initial object
        self._q_size = []  # list of PIDs we pushed, so this is atomic.

    def empty(self):
        return self._q.empty() and (not self._internal_q or self._internal_q.empty())

    def is_pending(self):
        # check if we have pending requests to be pushed (it does not mean they were pulled)
        # only call from main put process
        return self._get_q_size_len() > 0

    def close(self, event, timeout=3.0):
        # wait until all pending requests pushed
        tic = time()
        pid = os.getpid()
        prev_q_size = self._get_q_size_len(pid)
        while self.is_pending():
            if event:
                event.set()
            if not self.__thread_pool.is_active():
                break
            sleep(0.1)
            # timeout is for the maximum time to pull a single object from the queue,
            # this way if we get stuck we notice quickly and abort
            if timeout and (time()-tic) > timeout:
                if prev_q_size == self._get_q_size_len(pid):
                    break
                else:
                    prev_q_size = self._get_q_size_len(pid)
                    tic = time()

    def get(self, *args, **kwargs):
        return self._get_internal_queue(*args, **kwargs)

    def batch_get(self, max_items=1000, timeout=0.2, throttle_sleep=0.1):
        buffer = []
        timeout_count = int(timeout/throttle_sleep)
        empty_count = timeout_count
        while len(buffer) < max_items:
            while not self.empty() and len(buffer) < max_items:
                try:
                    buffer.append(self._get_internal_queue(block=False))
                    empty_count = 0
                except Empty:
                    break
            empty_count += 1
            if empty_count > timeout_count or len(buffer) >= max_items:
                break
            sleep(throttle_sleep)
        return buffer

    def put(self, obj):
        # not atomic when forking for the first time
        # GIL will make sure it is atomic
        self._q_size.append(os.getpid())
        try:
            # make sure the block put is done in the thread pool i.e. in the background
            obj = pickle.dumps(obj)
            if BackgroundMonitor.get_at_exit_state():
                self._q_put(obj)
                return
            self.__thread_pool.get().apply_async(self._q_put, args=(obj, ))
        except:  # noqa
            pid = os.getpid()
            p = None
            while p != pid and self._q_size:
                p = self._q_size.pop()

    def _get_q_size_len(self, pid=None):
        pid = pid or os.getpid()
        return len([p for p in self._q_size if p == pid])

    def _q_put(self, obj):
        try:
            self._q.put(obj)
        except BaseException:
            # make sure we zero the _q_size of the process dies (i.e. queue put fails)
            self._q_size.clear()
            raise
        pid = os.getpid()
        # GIL will make sure it is atomic
        # pop the First "counter" that is ours (i.e. pid == os.getpid())
        p = None
        while p != pid and self._q_size:
            p = self._q_size.pop()

    def _init_reader_thread(self):
        if not self._internal_q:
            self._internal_q = ForkQueue()
        if not self._reader_thread or not self._reader_thread.is_alive():
            # read before we start the thread
            self._reader_thread = Thread(target=self._reader_daemon)
            self._reader_thread.daemon = True
            self._reader_thread.start()
            # if we have waiting results
            # wait until thread is up and pushed some results
            while not self._reader_thread_started:
                sleep(0.2)
            # just in case make sure we pulled some stuff if we had any
            # todo: wait until a queue is not empty, but for some reason that might fail
            sleep(1.0)

    def _get_internal_queue(self, *args, **kwargs):
        self._init_reader_thread()
        obj = self._internal_q.get(*args, **kwargs)
        # deserialize
        return pickle.loads(obj)

    def _reader_daemon(self):
        self._reader_thread_started = True
        # pull from process queue and push into thread queue
        while True:
            # noinspection PyBroadException
            try:
                obj = self._q.get()
                if obj is None:
                    break
            except Exception:
                break
            self._internal_q.put(obj)

    @staticmethod
    def _pipe_override_send_bytes(self, buf):
        n = len(buf)
        # For wire compatibility with 3.2 and lower
        header = struct.pack("!i", n)
        # Issue #20540: concatenate before sending, to avoid delays due
        # to Nagle's algorithm on a TCP socket.
        # Also note we want to avoid sending a 0-length buffer separately,
        # to avoid "broken pipe" errors if the other end closed the pipe.
        self._send(header + buf)


class SafeEvent(object):
    __thread_pool = SingletonThreadPool()

    def __init__(self):
        self._event = ProcessEvent()

    def is_set(self):
        return self._event.is_set()

    def set(self):
        if not BackgroundMonitor.is_subprocess_enabled() or BackgroundMonitor.is_subprocess_alive():
            self._event.set()
        # SafeEvent.__thread_pool.get().apply_async(func=self._event.set, args=())

    def clear(self):
        return self._event.clear()

    def wait(self, timeout=None):
        return self._event.wait(timeout=timeout)


class SingletonLock(AbstractContextManager):
    _instances = []

    def __init__(self):
        self._lock = None
        SingletonLock._instances.append(self)

    def acquire(self, *args, **kwargs):
        self.create()
        return self._lock.acquire(*args, **kwargs)

    def release(self, *args, **kwargs):
        if self._lock is None:
            return None
        return self._lock.release(*args, **kwargs)

    def create(self):
        if self._lock is None:
            self._lock = ForkSafeRLock()

    @classmethod
    def instantiate(cls):
        for i in cls._instances:
            i.create()

    def __enter__(self):
        """Return `self` upon entering the runtime context."""
        self.acquire()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        """Raise any exception triggered within the runtime context."""
        # Do whatever cleanup.
        self.release()


class BackgroundMonitor(object):
    # If we need multiple monitoring contexts (i.e. subprocesses) this will become a dict
    _main_process = None
    _main_process_proc_obj = None
    _main_process_task_id = None
    _parent_pid = None
    _sub_process_started = None
    _at_exit = False
    _instances = {}  # type: Dict[int, List[BackgroundMonitor]]

    def __init__(self, task, wait_period):
        self._event = ForkEvent()
        self._done_ev = ForkEvent()
        self._start_ev = ForkEvent()
        self._task_pid = os.getpid()
        self._thread = None
        self._thread_pid = None
        self._wait_timeout = wait_period
        self._subprocess = None if task.is_main_task() else False
        self._task_id = task.id
        self._task_obj_id = id(task.id)

    def start(self):
        if not self._thread:
            self._thread = True
        self._event.clear()
        self._done_ev.clear()
        if self._subprocess is False:
            # start the thread we are in threading mode.
            self._start()
        else:
            # append to instances
            if self not in self._get_instances():
                self._get_instances().append(self)

    def wait(self, timeout=None):
        if not self._done_ev:
            return
        if not self.is_subprocess_mode() or self.is_subprocess_mode_and_parent_process():
            self._done_ev.wait(timeout=timeout)

    def _start(self):
        # if we already started do nothing
        if isinstance(self._thread, Thread):
            if self._thread_pid == os.getpid():
                return
        self._thread_pid = os.getpid()
        self._thread = Thread(target=self._daemon)
        self._thread.daemon = True
        self._thread.start()

    def stop(self):
        if not self._thread:
            return

        if not self._is_subprocess_mode_and_not_parent_process() and (
                not self.is_subprocess_mode() or self.is_subprocess_alive()):
            self._event.set()

        if isinstance(self._thread, Thread):
            try:
                self._get_instances().remove(self)
            except ValueError:
                pass
            self._thread = False

    def daemon(self):
        while True:
            if self._event.wait(self._wait_timeout):
                break
            self._daemon_step()

    def _daemon(self):
        self._start_ev.set()
        try:
            self.daemon()
        finally:
            self.post_execution()
            self._thread = False

    def post_execution(self):
        self._done_ev.set()

    def set_subprocess_mode(self):
        # called just before launching the daemon in a subprocess
        if not self._subprocess:
            self._subprocess = True
        if not isinstance(self._done_ev, SafeEvent):
            self._done_ev = SafeEvent()
        if not isinstance(self._start_ev, SafeEvent):
            self._start_ev = SafeEvent()
        if not isinstance(self._event, SafeEvent):
            self._event = SafeEvent()

    def _daemon_step(self):
        pass

    @classmethod
    def start_all(cls, task, wait_for_subprocess=True):
        # noinspection PyProtectedMember
        execute_in_subprocess = task._report_subprocess_enabled

        if not execute_in_subprocess:
            for d in BackgroundMonitor._instances.get(id(task.id), []):
                d._start()
        elif not BackgroundMonitor._main_process:
            cls._parent_pid = os.getpid()
            cls._sub_process_started = SafeEvent()
            cls._sub_process_started.clear()
            cls._main_process_task_id = task.id
            # setup
            for d in BackgroundMonitor._instances.get(id(task.id), []):
                d.set_subprocess_mode()

            # ToDo: solve for standalone spawn subprocess
            # prefer os.fork, because multipprocessing.Process add atexit callback, which might later be invalid.
            cls.__start_subprocess_os_fork(task_obj_id=id(task.id))
            # if ForkContext is not None and isinstance(get_context(), ForkContext):
            #     cls.__start_subprocess_forkprocess(task_obj_id=id(task.id))
            # else:
            #     cls.__start_subprocess_os_fork(task_obj_id=id(task.id))

            # wait until subprocess is up
            if wait_for_subprocess:
                cls._sub_process_started.wait()

    @classmethod
    def __start_subprocess_os_fork(cls, task_obj_id):
        process_args = (task_obj_id, cls._sub_process_started, os.getpid())
        BackgroundMonitor._main_process = os.fork()
        # check if we are the child process
        if BackgroundMonitor._main_process == 0:
            # update to the child process pid
            BackgroundMonitor._main_process = os.getpid()
            BackgroundMonitor._main_process_proc_obj = psutil.Process(BackgroundMonitor._main_process)
            cls._background_process_start(*process_args)
            # force to leave the subprocess
            leave_process(0)
            return

        # update main process object (we are now in the parent process, and we update on the child's subprocess pid)
        # noinspection PyBroadException
        try:
            BackgroundMonitor._main_process_proc_obj = psutil.Process(BackgroundMonitor._main_process)
        except Exception:
            # if we fail for some reason, do not crash, switch to thread mode when you can
            BackgroundMonitor._main_process_proc_obj = None

    @classmethod
    def __start_subprocess_forkprocess(cls, task_obj_id):
        _main_process = Process(
            target=cls._background_process_start,
            args=(task_obj_id, cls._sub_process_started, os.getpid())
        )
        _main_process.daemon = True
        # Hack allow to create daemon subprocesses (even though python doesn't like it)
        un_daemonize = False
        # noinspection PyBroadException
        try:
            from multiprocessing import current_process
            if current_process()._config.get('daemon'):  # noqa
                un_daemonize = current_process()._config.get('daemon')  # noqa
                current_process()._config['daemon'] = False  # noqa
        except BaseException:
            pass
        # try to start the background process, if we fail retry again, or crash
        for i in range(4):
            try:
                _main_process.start()
                break
            except BaseException:
                if i < 3:
                    sleep(1)
                    continue
                raise
        BackgroundMonitor._main_process = _main_process.pid
        BackgroundMonitor._main_process_proc_obj = psutil.Process(BackgroundMonitor._main_process)
        if un_daemonize:
            # noinspection PyBroadException
            try:
                from multiprocessing import current_process
                current_process()._config['daemon'] = un_daemonize  # noqa
            except BaseException:
                pass

    @classmethod
    def _background_process_start(cls, task_obj_id, event_start=None, parent_pid=None):
        # type: (int, Optional[SafeEvent], Optional[int]) -> None
        is_debugger_running = bool(getattr(sys, 'gettrace', None) and sys.gettrace())
        # make sure we update the pid to our own
        cls._main_process = os.getpid()
        cls._main_process_proc_obj = psutil.Process(cls._main_process)
        # restore original signal, this will prevent any deadlocks
        # Do not change the exception we need to catch base exception as well
        # noinspection PyBroadException
        try:
            from ... import Task
            # make sure we do not call Task.current_task() it will create a Task object for us on a subprocess!
            # noinspection PyProtectedMember
            if Task._has_current_task_obj():
                # noinspection PyProtectedMember
                Task.current_task()._remove_at_exit_callbacks()
        except:  # noqa
            pass

        # if a debugger is running, wait for it to attach to the subprocess
        if is_debugger_running:
            sleep(3)

        instances = BackgroundMonitor._instances.get(task_obj_id, [])
        # launch all the threads
        for d in instances:
            d._start()

        if cls._sub_process_started:
            cls._sub_process_started.set()

        if event_start:
            event_start.set()

        # wait until we are signaled
        for i in instances:
            # DO NOT CHANGE, we need to catch base exception, if the process gte's killed
            try:
                while i._thread is None or (i._thread and i._thread.is_alive()):
                    # thread is still not up
                    if i._thread is None:
                        sleep(0.1)
                        continue

                    # noinspection PyBroadException
                    try:
                        p = psutil.Process(parent_pid)
                        parent_alive = p.is_running() and p.status() != psutil.STATUS_ZOMBIE
                    except Exception:
                        parent_alive = False

                    # if parent process is not here we should just leave!
                    if not parent_alive:
                        return

                    # DO NOT CHANGE, we need to catch base exception, if the process gte's killed
                    try:
                        # timeout so we can detect if the parent process got killed.
                        i._thread.join(timeout=30.)
                    except:  # noqa
                        break
            except:  # noqa
                pass
        # we are done, leave process
        return

    def is_alive(self):
        if not self.is_subprocess_mode():
            return isinstance(self._thread, Thread) and self._thread.is_alive()

        if self.get_at_exit_state():
            return self.is_subprocess_alive() and self._thread

        return self.is_subprocess_alive() and \
            self._thread and \
            self._start_ev.is_set() and \
            not self._done_ev.is_set()

    @classmethod
    def _fast_is_subprocess_alive(cls):
        if not cls._main_process_proc_obj:
            return False
        # we have to assume the process actually exists, so we optimize for
        # just getting the object and status.
        # noinspection PyBroadException
        try:
            return cls._main_process_proc_obj.is_running() and \
                   cls._main_process_proc_obj.status() != psutil.STATUS_ZOMBIE
        except Exception:
            return False

    @classmethod
    def is_subprocess_alive(cls, task=None):
        if not cls._main_process or (task and cls._main_process_task_id != task.id):
            return False
        # noinspection PyBroadException
        try:
            p = psutil.Process(cls._main_process)
            return p.is_running() and p.status() != psutil.STATUS_ZOMBIE
        except Exception:
            current_pid = cls._main_process
            if not current_pid:
                return False
            try:
                parent = psutil.Process(cls._parent_pid)
            except psutil.Error:
                # could not find parent process id
                return
            for child in parent.children(recursive=True):
                # kill ourselves last (if we need to)
                if child.pid == current_pid:
                    return child.is_running() and child.status() != psutil.STATUS_ZOMBIE
            return False

    def is_subprocess_mode(self):
        return self._subprocess is not False and \
               bool(self._main_process) and self._task_id == self._main_process_task_id

    def _get_instances(self):
        return self._instances.setdefault(self._task_obj_id, [])

    def _is_subprocess_mode_and_not_parent_process(self):
        return self.is_subprocess_mode() and self._parent_pid != os.getpid()

    def is_subprocess_mode_and_parent_process(self):
        return self.is_subprocess_mode() and self._parent_pid == os.getpid()

    def _is_thread_mode_and_not_main_process(self):
        if self.is_subprocess_mode():
            return False
        from ... import Task
        # noinspection PyProtectedMember
        return Task._Task__is_subprocess()

    @classmethod
    def is_subprocess_enabled(cls, task=None):
        return bool(cls._main_process) and (not task or task.id == cls._main_process_task_id)

    @classmethod
    def clear_main_process(cls, task):
        if BackgroundMonitor._main_process_task_id != task.id:
            return
        cls.wait_for_sub_process(task)
        BackgroundMonitor._main_process = None
        BackgroundMonitor._main_process_proc_obj = None
        BackgroundMonitor._main_process_task_id = None
        BackgroundMonitor._parent_pid = None
        BackgroundMonitor._sub_process_started = None
        BackgroundMonitor._instances = {}
        SingletonThreadPool.clear()

    @classmethod
    def wait_for_sub_process(cls, task, timeout=None):
        if not cls.is_subprocess_enabled(task=task):
            return

        for d in BackgroundMonitor._instances.get(id(task.id), []):
            d.stop()

        tic = time()
        while cls.is_subprocess_alive(task=task) and (not timeout or time()-tic < timeout):
            sleep(0.03)

    @classmethod
    def set_at_exit_state(cls, state=True):
        cls._at_exit = bool(state)

    @classmethod
    def get_at_exit_state(cls):
        return cls._at_exit


def leave_process(status=0):
    # type: (int) -> None
    """
    Exit current process with status-code (status)
    :param status: int exit code
    """
    try:
        sys.exit(status or 0)
    except:   # noqa
        # ipython/jupyter notebook will not allow to call sys.exit
        # we have to call the low level function
        os._exit(status or 0)  # noqa


# clearml/utilities/pyhocon/__init__.py

from .config_parser import ConfigParser, ConfigFactory, ConfigMissingException
from .config_tree import ConfigTree
from .converter import HOCONConverter

__all__ = ["ConfigParser", "ConfigFactory", "ConfigMissingException", "ConfigTree", "HOCONConverter"]


# clearml/utilities/pyhocon/config_parser.py

import itertools
import re
import os
import socket
import contextlib
import codecs
from datetime import timedelta

from pyparsing import Forward, Keyword, QuotedString, Word, Literal, Suppress, Regex, Optional, SkipTo, ZeroOrMore, \
    Group, lineno, col, TokenConverter, replaceWith, alphanums, alphas8bit, ParseSyntaxException, StringEnd
from pyparsing import ParserElement
from .config_tree import ConfigTree, ConfigSubstitution, ConfigList, ConfigValues, ConfigUnquotedString, \
    ConfigInclude, NoneValue, ConfigQuotedString
from .exceptions import ConfigSubstitutionException, ConfigMissingException, ConfigException
import logging
import copy

use_urllib2 = False
try:
    # For Python 3.0 and later
    from urllib.request import urlopen
    from urllib.error import HTTPError, URLError
except ImportError:  # pragma: no cover
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen, HTTPError, URLError

    use_urllib2 = True
try:
    basestring
except NameError:  # pragma: no cover
    basestring = str
    unicode = str

logger = logging.getLogger(__name__)

#
# Substitution Defaults
#


class DEFAULT_SUBSTITUTION(object):
    pass


class MANDATORY_SUBSTITUTION(object):
    pass


class NO_SUBSTITUTION(object):
    pass


class STR_SUBSTITUTION(object):
    pass


def period(period_value, period_unit):
    try:
        from dateutil.relativedelta import relativedelta as period_impl
    except Exception:
        from datetime import timedelta as period_impl

    if period_unit == 'nanoseconds':
        period_unit = 'microseconds'
        period_value = int(period_value / 1000)

    arguments = dict(zip((period_unit,), (period_value,)))

    if period_unit == 'milliseconds':
        return timedelta(**arguments)

    return period_impl(**arguments)


class ConfigFactory(object):

    @classmethod
    def parse_file(cls, filename, encoding='utf-8', required=True, resolve=True, unresolved_value=DEFAULT_SUBSTITUTION):
        """Parse file

        :param filename: filename
        :type filename: basestring
        :param encoding: file encoding
        :type encoding: basestring
        :param required: If true, raises an exception if can't load file
        :type required: boolean
        :param resolve: if true, resolve substitutions
        :type resolve: boolean
        :param unresolved_value: assigned value value to unresolved substitution.
            If overriden with a default value, it will replace all unresolved value to the default value.
            If it is set to to pyhocon.STR_SUBSTITUTION then it will replace the value by its
            substitution expression (e.g., ${x})
        :type unresolved_value: class
        :return: Config object
        :type return: Config
        """
        try:
            with codecs.open(filename, 'r', encoding=encoding) as fd:
                content = fd.read()
                return cls.parse_string(content, os.path.dirname(filename), resolve, unresolved_value)
        except IOError as e:
            if required:
                raise e
            logger.warn('Cannot include file %s. File does not exist or cannot be read.', filename)
            return []

    @classmethod
    def parse_URL(cls, url, timeout=None, resolve=True, required=False, unresolved_value=DEFAULT_SUBSTITUTION):
        """Parse URL

        :param url: url to parse
        :type url: basestring
        :param resolve: if true, resolve substitutions
        :type resolve: boolean
        :param unresolved_value: assigned value value to unresolved substitution.
            If overriden with a default value, it will replace all unresolved value to the default value.
            If it is set to to pyhocon.STR_SUBSTITUTION then it will replace the value by
            its substitution expression (e.g., ${x})
        :type unresolved_value: boolean
        :return: Config object or []
        :type return: Config or list
        """
        socket_timeout = socket._GLOBAL_DEFAULT_TIMEOUT if timeout is None else timeout

        try:
            with contextlib.closing(urlopen(url, timeout=socket_timeout)) as fd:
                content = fd.read() if use_urllib2 else fd.read().decode('utf-8')
                return cls.parse_string(content, os.path.dirname(url), resolve, unresolved_value)
        except (HTTPError, URLError) as e:
            logger.warn('Cannot include url %s. Resource is inaccessible.', url)
            if required:
                raise e
            else:
                return []

    @classmethod
    def parse_string(cls, content, basedir=None, resolve=True, unresolved_value=DEFAULT_SUBSTITUTION):
        """Parse URL

        :param content: content to parse
        :type content: basestring
        :param resolve: If true, resolve substitutions
        :param resolve: if true, resolve substitutions
        :type resolve: boolean
        :param unresolved_value: assigned value value to unresolved substitution.
            If overriden with a default value, it will replace all unresolved value to the default value.
            If it is set to to pyhocon.STR_SUBSTITUTION then it will replace the value by
            its substitution expression (e.g., ${x})
        :type unresolved_value: boolean
        :return: Config object
        :type return: Config
        """
        return ConfigParser().parse(content, basedir, resolve, unresolved_value)

    @classmethod
    def from_dict(cls, dictionary, root=False):
        """Convert dictionary (and ordered dictionary) into a ConfigTree
        :param dictionary: dictionary to convert
        :type dictionary: dict
        :return: Config object
        :type return: Config
        """

        def create_tree(value):
            if isinstance(value, dict):
                res = ConfigTree(root=root)
                for key, child_value in value.items():
                    res.put(key, create_tree(child_value))
                return res
            if isinstance(value, list):
                return [create_tree(v) for v in value]
            else:
                return value

        return create_tree(dictionary)


class ConfigParser(object):
    """
    Parse HOCON files: https://github.com/typesafehub/config/blob/master/HOCON.md
    """

    REPLACEMENTS = {
        '\\\\': '\\',
        '\\\n': '\n',
        '\\n': '\n',
        '\\r': '\r',
        '\\t': '\t',
        '\\=': '=',
        '\\#': '#',
        '\\!': '!',
        '\\"': '"',
    }

    period_type_map = {
        'nanoseconds': ['ns', 'nano', 'nanos', 'nanosecond', 'nanoseconds'],

        'microseconds': ['us', 'micro', 'micros', 'microsecond', 'microseconds'],
        'milliseconds': ['ms', 'milli', 'millis', 'millisecond', 'milliseconds'],
        'seconds': ['s', 'second', 'seconds'],
        'minutes': ['m', 'minute', 'minutes'],
        'hours': ['h', 'hour', 'hours'],
        'weeks': ['w', 'week', 'weeks'],
        'days': ['d', 'day', 'days'],

    }

    optional_period_type_map = {
        'months': ['mo', 'month', 'months'],  # 'm' from hocon spec removed. conflicts with minutes syntax.
        'years': ['y', 'year', 'years']
    }

    supported_period_map = None

    @classmethod
    def get_supported_period_type_map(cls):
        if cls.supported_period_map is None:
            cls.supported_period_map = {}
            cls.supported_period_map.update(cls.period_type_map)

            try:
                from dateutil import relativedelta

                if relativedelta is not None:
                    cls.supported_period_map.update(cls.optional_period_type_map)
            except Exception:
                pass

        return cls.supported_period_map

    @classmethod
    def parse(cls, content, basedir=None, resolve=True, unresolved_value=DEFAULT_SUBSTITUTION):
        """parse a HOCON content

        :param content: HOCON content to parse
        :type content: basestring
        :param resolve: if true, resolve substitutions
        :type resolve: boolean
        :param unresolved_value: assigned value value to unresolved substitution.
            If overriden with a default value, it will replace all unresolved value to the default value.
            If it is set to to pyhocon.STR_SUBSTITUTION then it will replace the value by
            its substitution expression (e.g., ${x})
        :type unresolved_value: boolean
        :return: a ConfigTree or a list
        """

        unescape_pattern = re.compile(r'\\.')

        def replace_escape_sequence(match):
            value = match.group(0)
            return cls.REPLACEMENTS.get(value, value)

        def norm_string(value):
            return unescape_pattern.sub(replace_escape_sequence, value)

        def unescape_string(tokens):
            return ConfigUnquotedString(norm_string(tokens[0]))

        def parse_multi_string(tokens):
            # remove the first and last 3 "
            return tokens[0][3: -3]

        def convert_number(tokens):
            n = tokens[0]
            try:
                return int(n, 10)
            except ValueError:
                return float(n)

        def safe_convert_number(tokens):
            n = tokens[0]
            try:
                return int(n, 10)
            except ValueError:
                try:
                    return float(n)
                except ValueError:
                    return n

        def convert_period(tokens):

            period_value = int(tokens.value)
            period_identifier = tokens.unit

            period_unit = next((single_unit for single_unit, values
                                in cls.get_supported_period_type_map().items()
                                if period_identifier in values))

            return period(period_value, period_unit)

        # ${path} or ${?path} for optional substitution
        SUBSTITUTION_PATTERN = r"\$\{(?P<optional>\?)?(?P<variable>[^}]+)\}(?P<ws>[ \t]*)"

        def create_substitution(instring, loc, token):
            # remove the ${ and }
            match = re.match(SUBSTITUTION_PATTERN, token[0])
            variable = match.group('variable')
            ws = match.group('ws')
            optional = match.group('optional') == '?'
            substitution = ConfigSubstitution(variable, optional, ws, instring, loc)
            return substitution

        # ${path} or ${?path} for optional substitution
        STRING_PATTERN = '"(?P<value>(?:[^"\\\\]|\\\\.)*)"(?P<ws>[ \t]*)'

        def create_quoted_string(instring, loc, token):
            # remove the ${ and }
            match = re.match(STRING_PATTERN, token[0])
            value = norm_string(match.group('value'))
            ws = match.group('ws')
            return ConfigQuotedString(value, ws, instring, loc)

        def include_config(instring, loc, token):
            url = None
            file = None
            required = False

            if token[0] == 'required':
                required = True
                final_tokens = token[1:]
            else:
                final_tokens = token

            if len(final_tokens) == 1:  # include "test"
                value = final_tokens[0].value if isinstance(final_tokens[0], ConfigQuotedString) else final_tokens[0]
                if value.startswith("http://") or value.startswith("https://") or value.startswith("file://"):
                    url = value
                else:
                    file = value
            elif len(final_tokens) == 2:  # include url("test") or file("test")
                value = final_tokens[1].value if isinstance(token[1], ConfigQuotedString) else final_tokens[1]
                if final_tokens[0] == 'url':
                    url = value
                else:
                    file = value

            if url is not None:
                logger.debug('Loading config from url %s', url)
                obj = ConfigFactory.parse_URL(
                    url,
                    resolve=False,
                    required=required,
                    unresolved_value=NO_SUBSTITUTION
                )
            elif file is not None:
                path = file if basedir is None else os.path.join(basedir, file)
                logger.debug('Loading config from file %s', path)
                obj = ConfigFactory.parse_file(
                    path,
                    resolve=False,
                    required=required,
                    unresolved_value=NO_SUBSTITUTION
                )
            else:
                raise ConfigException('No file or URL specified at: {loc}: {instring}', loc=loc, instring=instring)

            return ConfigInclude(obj if isinstance(obj, list) else obj.items())

        @contextlib.contextmanager
        def set_default_white_spaces():
            default = ParserElement.DEFAULT_WHITE_CHARS
            ParserElement.setDefaultWhitespaceChars(' \t')
            yield
            ParserElement.setDefaultWhitespaceChars(default)

        with set_default_white_spaces():
            assign_expr = Forward()
            true_expr = Keyword("true", caseless=True).setParseAction(replaceWith(True))
            false_expr = Keyword("false", caseless=True).setParseAction(replaceWith(False))
            null_expr = Keyword("null", caseless=True).setParseAction(replaceWith(NoneValue()))
            # key = QuotedString('"', escChar='\\', unquoteResults=False) | Word(alphanums + alphas8bit + '._- /')
            regexp_numbers = r'[+-]?(\d*\.\d+|\d+(\.\d+)?)([eE][+\-]?\d+)?(?=$|[ \t]*([\$\}\],#\n\r]|//))'
            key = QuotedString('"', escChar='\\', unquoteResults=False) | \
                Regex(regexp_numbers, re.DOTALL).setParseAction(safe_convert_number) | \
                Word(alphanums + alphas8bit + '._- /')

            eol = Word('\n\r').suppress()
            eol_comma = Word('\n\r,').suppress()
            comment = (Literal('#') | Literal('//')) - SkipTo(eol | StringEnd())
            comment_eol = Suppress(Optional(eol_comma) + comment)
            comment_no_comma_eol = (comment | eol).suppress()
            number_expr = Regex(regexp_numbers, re.DOTALL).setParseAction(convert_number)

            period_types = itertools.chain.from_iterable(cls.get_supported_period_type_map().values())
            period_expr = Regex(r'(?P<value>\d+)\s*(?P<unit>' + '|'.join(period_types) + ')$'
                                ).setParseAction(convert_period)

            # multi line string using """
            # Using fix described in http://pyparsing.wikispaces.com/share/view/3778969
            multiline_string = Regex('""".*?"*"""', re.DOTALL | re.UNICODE).setParseAction(parse_multi_string)
            # single quoted line string
            quoted_string = Regex(r'"(?:[^"\\\n]|\\.)*"[ \t]*', re.UNICODE).setParseAction(create_quoted_string)
            # unquoted string that takes the rest of the line until an optional comment
            # we support .properties multiline support which is like this:
            # line1  \
            # line2 \
            # so a backslash precedes the \n
            unquoted_string = Regex(r'(?:[^^`+?!@*&"\[\{\s\]\}#,=\$\\]|\\.)+[ \t]*',
                                    re.UNICODE).setParseAction(unescape_string)
            substitution_expr = Regex(r'[ \t]*\$\{[^\}]+\}[ \t]*').setParseAction(create_substitution)
            string_expr = multiline_string | quoted_string | unquoted_string

            value_expr = period_expr | number_expr | true_expr | false_expr | null_expr | string_expr

            include_content = (quoted_string | ((Keyword('url') | Keyword(
                'file')) - Literal('(').suppress() - quoted_string - Literal(')').suppress()))
            include_expr = (
                Keyword("include", caseless=True).suppress() + (
                    include_content | (
                        Keyword("required") - Literal('(').suppress() - include_content - Literal(')').suppress()
                    )
                )
            ).setParseAction(include_config)

            root_dict_expr = Forward()
            dict_expr = Forward()
            list_expr = Forward()
            multi_value_expr = ZeroOrMore(comment_eol | include_expr | substitution_expr |
                                          dict_expr | list_expr | value_expr | (Literal('\\') - eol).suppress())
            # for a dictionary : or = is optional
            # last zeroOrMore is because we can have t = {a:4} {b: 6} {c: 7} which is dictionary concatenation
            inside_dict_expr = ConfigTreeParser(ZeroOrMore(comment_eol | include_expr | assign_expr | eol_comma))
            inside_root_dict_expr = ConfigTreeParser(ZeroOrMore(
                comment_eol | include_expr | assign_expr | eol_comma), root=True)
            dict_expr << Suppress('{') - inside_dict_expr - Suppress('}')
            root_dict_expr << Suppress('{') - inside_root_dict_expr - Suppress('}')
            list_entry = ConcatenatedValueParser(multi_value_expr)
            list_expr << Suppress('[') - ListParser(list_entry - ZeroOrMore(eol_comma - list_entry)) - Suppress(']')

            # special case when we have a value assignment where the string can potentially be the remainder of the line
            assign_expr << Group(key - ZeroOrMore(comment_no_comma_eol) -
                                 (dict_expr | (Literal('=') | Literal(':') | Literal('+=')) -
                                  ZeroOrMore(comment_no_comma_eol) - ConcatenatedValueParser(multi_value_expr)))

            # the file can be { ... } where {} can be omitted or []
            config_expr = ZeroOrMore(comment_eol | eol) + (list_expr | root_dict_expr |
                                                           inside_root_dict_expr) + ZeroOrMore(comment_eol | eol_comma)
            config = config_expr.parseString(content, parseAll=True)[0]

            if resolve:
                allow_unresolved = resolve and unresolved_value is not DEFAULT_SUBSTITUTION and \
                                   unresolved_value is not MANDATORY_SUBSTITUTION
                has_unresolved = cls.resolve_substitutions(config, allow_unresolved)
                if has_unresolved and unresolved_value is MANDATORY_SUBSTITUTION:
                    raise ConfigSubstitutionException(
                        'resolve cannot be set to True and unresolved_value to MANDATORY_SUBSTITUTION')

            if unresolved_value is not NO_SUBSTITUTION and unresolved_value is not DEFAULT_SUBSTITUTION:
                cls.unresolve_substitutions_to_value(config, unresolved_value)
        return config

    @classmethod
    def _resolve_variable(cls, config, substitution):
        """
        :param config:
        :param substitution:
        :return: (is_resolved, resolved_variable)
        """
        variable = substitution.variable
        try:
            return True, config.get(variable)
        except ConfigMissingException:
            # default to environment variable
            value = os.environ.get(variable)

            if value is None:
                if substitution.optional:
                    return False, None
                else:
                    raise ConfigSubstitutionException(
                        "Cannot resolve variable ${{{variable}}} (line: {line}, col: {col})".format(
                            variable=variable,
                            line=lineno(substitution.loc, substitution.instring),
                            col=col(substitution.loc, substitution.instring)))
            elif isinstance(value, ConfigList) or isinstance(value, ConfigTree):
                raise ConfigSubstitutionException(
                    "Cannot substitute variable ${{{variable}}} because it does not point to a "
                    "string, int, float, boolean or null {type} (line:{line}, col: {col})".format(
                        variable=variable,
                        type=value.__class__.__name__,
                        line=lineno(substitution.loc, substitution.instring),
                        col=col(substitution.loc, substitution.instring)))
            return True, value

    @classmethod
    def _fixup_self_references(cls, config, accept_unresolved=False):
        if isinstance(config, ConfigTree) and config.root:
            for key in config:  # Traverse history of element
                history = config.history[key]
                previous_item = history[0]
                for current_item in history[1:]:
                    for substitution in cls._find_substitutions(current_item):
                        prop_path = ConfigTree.parse_key(substitution.variable)
                        if len(prop_path) > 1 and config.get(substitution.variable, None) is not None:
                            continue  # If value is present in latest version, don't do anything
                        if prop_path[0] == key:
                            if isinstance(previous_item, ConfigValues) and not accept_unresolved:
                                # We hit a dead end, we cannot evaluate
                                raise ConfigSubstitutionException(
                                    "Property {variable} cannot be substituted. Check for cycles.".format(
                                        variable=substitution.variable
                                    )
                                )
                            else:
                                value = previous_item if len(
                                    prop_path) == 1 else previous_item.get(".".join(prop_path[1:]))
                                _, _, current_item = cls._do_substitute(substitution, value)
                    previous_item = current_item

                if len(history) == 1:
                    for substitution in cls._find_substitutions(previous_item):
                        prop_path = ConfigTree.parse_key(substitution.variable)
                        if len(prop_path) > 1 and config.get(substitution.variable, None) is not None:
                            continue  # If value is present in latest version, don't do anything
                        if prop_path[0] == key and substitution.optional:
                            cls._do_substitute(substitution, None)
                        if prop_path[0] == key:
                            value = os.environ.get(key)
                            if value is not None:
                                cls._do_substitute(substitution, value)
                                continue
                            if substitution.optional:  # special case, when self optional referencing without existing
                                cls._do_substitute(substitution, None)

    # traverse config to find all the substitutions
    @classmethod
    def _find_substitutions(cls, item):
        """Convert HOCON input into a JSON output

        :return: JSON string representation
        :type return: basestring
        """
        if isinstance(item, ConfigValues):
            return item.get_substitutions()

        substitutions = []
        elements = []
        if isinstance(item, ConfigTree):
            elements = item.values()
        elif isinstance(item, list):
            elements = item

        for child in elements:
            substitutions += cls._find_substitutions(child)
        return substitutions

    @classmethod
    def _do_substitute(cls, substitution, resolved_value, is_optional_resolved=True):
        unresolved = False
        new_substitutions = []
        if isinstance(resolved_value, ConfigValues):
            resolved_value = resolved_value.transform()
        if isinstance(resolved_value, ConfigValues):
            unresolved = True
            result = resolved_value
        else:
            # replace token by substitution
            config_values = substitution.parent
            # if it is a string, then add the extra ws that was present in the original string after the substitution
            formatted_resolved_value = resolved_value \
                if resolved_value is None \
                or isinstance(resolved_value, (dict, list)) \
                or substitution.index == len(config_values.tokens) - 1 \
                else (str(resolved_value) + substitution.ws)
            # use a deepcopy of resolved_value to avoid mutation
            config_values.put(substitution.index, copy.deepcopy(formatted_resolved_value))
            transformation = config_values.transform()
            result = config_values.overriden_value \
                if transformation is None and not is_optional_resolved \
                else transformation

            if result is None and config_values.key in config_values.parent:
                del config_values.parent[config_values.key]
            else:
                config_values.parent[config_values.key] = result
                s = cls._find_substitutions(result)
                if s:
                    new_substitutions = s
                    unresolved = True

        return (unresolved, new_substitutions, result)

    @classmethod
    def _final_fixup(cls, item):
        if isinstance(item, ConfigValues):
            return item.transform()
        elif isinstance(item, list):
            return list([cls._final_fixup(child) for child in item])
        elif isinstance(item, ConfigTree):
            items = list(item.items())
            for key, child in items:
                item[key] = cls._final_fixup(child)
        return item

    @classmethod
    def unresolve_substitutions_to_value(cls, config, unresolved_value=STR_SUBSTITUTION):
        for substitution in cls._find_substitutions(config):
            if unresolved_value is STR_SUBSTITUTION:
                value = substitution.raw_str()
            elif unresolved_value is None:
                value = NoneValue()
            else:
                value = unresolved_value
            cls._do_substitute(substitution, value, False)
        cls._final_fixup(config)

    @classmethod
    def resolve_substitutions(cls, config, accept_unresolved=False):
        has_unresolved = False
        cls._fixup_self_references(config, accept_unresolved)
        substitutions = cls._find_substitutions(config)
        if len(substitutions) > 0:
            unresolved = True
            any_unresolved = True
            _substitutions = []
            cache = {}
            while any_unresolved and len(substitutions) > 0 and set(substitutions) != set(_substitutions):
                unresolved = False
                any_unresolved = True
                _substitutions = substitutions[:]

                for substitution in _substitutions:
                    is_optional_resolved, resolved_value = cls._resolve_variable(config, substitution)

                    # if the substitution is optional
                    if not is_optional_resolved and substitution.optional:
                        resolved_value = None
                    if isinstance(resolved_value, ConfigValues):
                        parents = cache.get(resolved_value)
                        if parents is None:
                            parents = []
                            link = resolved_value
                            while isinstance(link, ConfigValues):
                                parents.append(link)
                                link = link.overriden_value
                            cache[resolved_value] = parents

                    if isinstance(resolved_value, ConfigValues) \
                       and substitution.parent in parents \
                       and hasattr(substitution.parent, 'overriden_value') \
                       and substitution.parent.overriden_value:

                        # self resolution, backtrack
                        resolved_value = substitution.parent.overriden_value

                    unresolved, new_substitutions, result = cls._do_substitute(
                        substitution, resolved_value, is_optional_resolved)
                    any_unresolved = unresolved or any_unresolved
                    substitutions.extend(new_substitutions)
                    if not isinstance(result, ConfigValues):
                        substitutions.remove(substitution)

            cls._final_fixup(config)
            if unresolved:
                has_unresolved = True
                if not accept_unresolved:
                    raise ConfigSubstitutionException("Cannot resolve {variables}. Check for cycles.".format(
                        variables=', '.join('${{{variable}}}: (line: {line}, col: {col})'.format(
                            variable=substitution.variable,
                            line=lineno(substitution.loc, substitution.instring),
                            col=col(substitution.loc, substitution.instring)) for substitution in substitutions)))

        cls._final_fixup(config)
        return has_unresolved


class ListParser(TokenConverter):
    """Parse a list [elt1, etl2, ...]
    """

    def __init__(self, expr=None):
        super(ListParser, self).__init__(expr)
        self.saveAsList = True

    def postParse(self, instring, loc, token_list):
        """Create a list from the tokens

        :param instring:
        :param loc:
        :param token_list:
        :return:
        """
        cleaned_token_list = [token for tokens in (token.tokens if isinstance(token, ConfigInclude) else [token]
                                                   for token in token_list if token != '')
                              for token in tokens]
        config_list = ConfigList(cleaned_token_list)
        return [config_list]


class ConcatenatedValueParser(TokenConverter):
    def __init__(self, expr=None):
        super(ConcatenatedValueParser, self).__init__(expr)
        self.parent = None
        self.key = None

    def postParse(self, instring, loc, token_list):
        config_values = ConfigValues(token_list, instring, loc)
        return [config_values.transform()]


class ConfigTreeParser(TokenConverter):
    """
    Parse a config tree from tokens
    """

    def __init__(self, expr=None, root=False):
        super(ConfigTreeParser, self).__init__(expr)
        self.root = root
        self.saveAsList = True

    def postParse(self, instring, loc, token_list):
        """Create ConfigTree from tokens

        :param instring:
        :param loc:
        :param token_list:
        :return:
        """
        config_tree = ConfigTree(root=self.root)
        for element in token_list:
            expanded_tokens = element.tokens if isinstance(element, ConfigInclude) else [element]

            for tokens in expanded_tokens:
                # key, value1 (optional), ...
                key = tokens[0].strip() if isinstance(tokens[0], (unicode, basestring)) else tokens[0]
                operator = '='
                if len(tokens) == 3 and tokens[1].strip() in [':', '=', '+=']:
                    operator = tokens[1].strip()
                    values = tokens[2:]
                elif len(tokens) == 2:
                    values = tokens[1:]
                else:
                    raise ParseSyntaxException("Unknown tokens {tokens} received".format(tokens=tokens))
                # empty string
                if len(values) == 0:
                    config_tree.put(key, '')
                else:
                    value = values[0]
                    if isinstance(value, list) and operator == "+=":
                        value = ConfigValues([ConfigSubstitution(key, True, '', False, loc), value], False, loc)
                        config_tree.put(key, value, False)
                    elif isinstance(value, unicode) and operator == "+=":
                        value = ConfigValues([ConfigSubstitution(key, True, '', True, loc), ' ' + value], True, loc)
                        config_tree.put(key, value, False)
                    elif isinstance(value, list):
                        config_tree.put(key, value, False)
                    else:
                        existing_value = config_tree.get(key, None)
                        if isinstance(value, ConfigTree) and not isinstance(existing_value, list):
                            # Only Tree has to be merged with tree
                            config_tree.put(key, value, True)
                        elif isinstance(value, ConfigValues):
                            conf_value = value
                            value.parent = config_tree
                            value.key = key
                            if isinstance(existing_value, list) or isinstance(existing_value, ConfigTree):
                                config_tree.put(key, conf_value, True)
                            else:
                                config_tree.put(key, conf_value, False)
                        else:
                            config_tree.put(key, value, False)
        return config_tree


# clearml/utilities/pyhocon/config_tree.py

from collections import OrderedDict
from pyparsing import lineno
from pyparsing import col
try:
    basestring
except NameError:  # pragma: no cover
    basestring = str
    unicode = str

import re
import copy
from .exceptions import ConfigException, ConfigWrongTypeException, ConfigMissingException


class UndefinedKey(object):
    pass


class NonExistentKey(object):
    pass


class NoneValue(object):
    pass


class ConfigTree(OrderedDict):
    KEY_SEP = '.'

    def __init__(self, *args, **kwds):
        self.root = kwds.pop('root') if 'root' in kwds else False
        if self.root:
            self.history = {}
        super(ConfigTree, self).__init__(*args, **kwds)
        for key, value in self.items():
            if isinstance(value, ConfigValues):
                value.parent = self
                value.index = key

    @staticmethod
    def merge_configs(a, b, copy_trees=False):
        """Merge config b into a

        :param a: target config
        :type a: ConfigTree
        :param b: source config
        :type b: ConfigTree
        :return: merged config a
        """
        for key, value in b.items():
            # if key is in both a and b and both values are dictionary then merge it otherwise override it
            if key in a and isinstance(a[key], ConfigTree) and isinstance(b[key], ConfigTree):
                if copy_trees:
                    a[key] = a[key].copy()
                ConfigTree.merge_configs(a[key], b[key], copy_trees=copy_trees)
            else:
                if isinstance(value, ConfigValues):
                    value.parent = a
                    value.key = key
                    if key in a:
                        value.overriden_value = a[key]
                a[key] = value
                if a.root:
                    if b.root:
                        a.history[key] = a.history.get(key, []) + b.history.get(key, [value])
                    else:
                        a.history[key] = a.history.get(key, []) + [value]

        return a

    def _put(self, key_path, value, append=False):
        key_elt = key_path[0]
        if len(key_path) == 1:
            # if value to set does not exist, override
            # if they are both configs then merge
            # if not then override
            if key_elt in self and isinstance(self[key_elt], ConfigTree) and isinstance(value, ConfigTree):
                if self.root:
                    new_value = ConfigTree.merge_configs(ConfigTree(), self[key_elt], copy_trees=True)
                    new_value = ConfigTree.merge_configs(new_value, value, copy_trees=True)
                    self._push_history(key_elt, new_value)
                    self[key_elt] = new_value
                else:
                    ConfigTree.merge_configs(self[key_elt], value)
            elif append:
                # If we have t=1
                # and we try to put t.a=5 then t is replaced by {a: 5}
                l_value = self.get(key_elt, None)
                if isinstance(l_value, ConfigValues):
                    l_value.tokens.append(value)
                    l_value.recompute()
                elif isinstance(l_value, ConfigTree) and isinstance(value, ConfigValues):
                    value.overriden_value = l_value
                    value.tokens.insert(0, l_value)
                    value.recompute()
                    value.parent = self
                    value.key = key_elt
                    self._push_history(key_elt, value)
                    self[key_elt] = value
                elif isinstance(l_value, list) and isinstance(value, ConfigValues):
                    self._push_history(key_elt, value)
                    value.overriden_value = l_value
                    value.parent = self
                    value.key = key_elt
                    self[key_elt] = value
                elif isinstance(l_value, list):
                    self[key_elt] = l_value + value
                    self._push_history(key_elt, l_value)
                elif l_value is None:
                    self._push_history(key_elt, value)
                    self[key_elt] = value

                else:
                    raise ConfigWrongTypeException(
                        u"Cannot concatenate the list {key}: {value} to {prev_value} of {type}".format(
                            key='.'.join(key_path),
                            value=value,
                            prev_value=l_value,
                            type=l_value.__class__.__name__)
                    )
            else:
                # if there was an override keep overide value
                if isinstance(value, ConfigValues):
                    value.parent = self
                    value.key = key_elt
                    value.overriden_value = self.get(key_elt, None)
                self._push_history(key_elt, value)
                self[key_elt] = value
        else:
            next_config_tree = super(ConfigTree, self).get(key_elt)
            if not isinstance(next_config_tree, ConfigTree):
                # create a new dictionary or overwrite a previous value
                next_config_tree = ConfigTree()
                self._push_history(key_elt, next_config_tree)
                self[key_elt] = next_config_tree
            next_config_tree._put(key_path[1:], value, append)

    def _push_history(self, key, value):
        if self.root:
            hist = self.history.get(key)
            if hist is None:
                hist = self.history[key] = []
            hist.append(value)

    def _get(self, key_path, key_index=0, default=UndefinedKey):
        key_elt = key_path[key_index]
        elt = super(ConfigTree, self).get(key_elt, UndefinedKey)

        if elt is UndefinedKey:
            if default is UndefinedKey:
                raise ConfigMissingException(u"No configuration setting found for key {key}".format(
                    key='.'.join(key_path[: key_index + 1])))
            else:
                return default

        if key_index == len(key_path) - 1:
            if isinstance(elt, NoneValue):
                return None
            elif isinstance(elt, list):
                return [None if isinstance(x, NoneValue) else x for x in elt]
            else:
                return elt
        elif isinstance(elt, ConfigTree):
            return elt._get(key_path, key_index + 1, default)
        else:
            if default is UndefinedKey:
                raise ConfigWrongTypeException(
                    u"{key} has type {type} rather than dict".format(key='.'.join(key_path[:key_index + 1]),
                                                                     type=type(elt).__name__))
            else:
                return default

    @staticmethod
    def parse_key(string):
        """
        Split a key into path elements:
        - a.b.c => a, b, c
        - a."b.c" => a, QuotedKey("b.c") if . is any of the special characters: $}[]:=+#`^?!@*&.
        - "a" => a
        - a.b."c" => a, b, c (special case)
        :param string: either string key (parse '.' as sub-key) or int / float as regular keys
        :return:
        """
        if isinstance(string, (int, float)):
            return [string]

        special_characters = '$}[]:=+#`^?!@*&.'
        tokens = re.findall(
            r'"[^"]+"|[^{special_characters}]+'.format(special_characters=re.escape(special_characters)),
            string)

        def contains_special_character(token):
            return any((c in special_characters) for c in token)

        return [token if contains_special_character(token) else token.strip('"') for token in tokens]

    def put(self, key, value, append=False):
        """Put a value in the tree (dot separated)

        :param key: key to use (dot separated). E.g., a.b.c
        :type key: basestring
        :param value: value to put
        """
        self._put(ConfigTree.parse_key(key), value, append)

    def get(self, key, default=UndefinedKey):
        """Get a value from the tree

        :param key: key to use (dot separated). E.g., a.b.c
        :type key: basestring
        :param default: default value if key not found
        :type default: object
        :return: value in the tree located at key
        """
        return self._get(ConfigTree.parse_key(key), 0, default)

    def get_string(self, key, default=UndefinedKey):
        """Return string representation of value found at key

        :param key: key to use (dot separated). E.g., a.b.c
        :type key: basestring
        :param default: default value if key not found
        :type default: basestring
        :return: string value
        :type return: basestring
        """
        value = self.get(key, default)
        if value is None:
            return None

        string_value = unicode(value)
        if isinstance(value, bool):
            string_value = string_value.lower()
        return string_value

    def pop(self, key, default=UndefinedKey):
        """Remove specified key and return the corresponding value.
        If key is not found, default is returned if given, otherwise ConfigMissingException is raised

        This method assumes the user wants to remove the last value in the chain so it parses via parse_key
        and pops the last value out of the dict.

        :param key: key to use (dot separated). E.g., a.b.c
        :type key: basestring
        :param default: default value if key not found
        :type default: object
        :param default: default value if key not found
        :return: value in the tree located at key
        """
        if default != UndefinedKey and key not in self:
            return default

        value = self.get(key, UndefinedKey)
        lst = ConfigTree.parse_key(key)
        parent = self.KEY_SEP.join(lst[0:-1])
        child = lst[-1]

        if parent:
            self.get(parent).__delitem__(child)
        else:
            self.__delitem__(child)
        return value

    def get_int(self, key, default=UndefinedKey):
        """Return int representation of value found at key

        :param key: key to use (dot separated). E.g., a.b.c
        :type key: basestring
        :param default: default value if key not found
        :type default: int
        :return: int value
        :type return: int
        """
        value = self.get(key, default)
        try:
            return int(value) if value is not None else None
        except (TypeError, ValueError):
            raise ConfigException(
                u"{key} has type '{type}' rather than 'int'".format(key=key, type=type(value).__name__))

    def get_float(self, key, default=UndefinedKey):
        """Return float representation of value found at key

        :param key: key to use (dot separated). E.g., a.b.c
        :type key: basestring
        :param default: default value if key not found
        :type default: float
        :return: float value
        :type return: float
        """
        value = self.get(key, default)
        try:
            return float(value) if value is not None else None
        except (TypeError, ValueError):
            raise ConfigException(
                u"{key} has type '{type}' rather than 'float'".format(key=key, type=type(value).__name__))

    def get_bool(self, key, default=UndefinedKey):
        """Return boolean representation of value found at key

        :param key: key to use (dot separated). E.g., a.b.c
        :type key: basestring
        :param default: default value if key not found
        :type default: bool
        :return: boolean value
        :type return: bool
        """

        # String conversions as per API-recommendations:
        # https://github.com/typesafehub/config/blob/master/HOCON.md#automatic-type-conversions
        bool_conversions = {
            None: None,
            'true': True, 'yes': True, 'on': True,
            'false': False, 'no': False, 'off': False
        }
        string_value = self.get_string(key, default)
        if string_value is not None:
            string_value = string_value.lower()
        try:
            return bool_conversions[string_value]
        except KeyError:
            raise ConfigException(
                u"{key} does not translate to a Boolean value".format(key=key))

    def get_list(self, key, default=UndefinedKey):
        """Return list representation of value found at key

        :param key: key to use (dot separated). E.g., a.b.c
        :type key: basestring
        :param default: default value if key not found
        :type default: list
        :return: list value
        :type return: list
        """
        value = self.get(key, default)
        if isinstance(value, list):
            return value
        elif isinstance(value, ConfigTree):
            lst = []
            for k, v in sorted(value.items(), key=lambda kv: kv[0]):
                if re.match('^[1-9][0-9]*$|0', k):
                    lst.append(v)
                else:
                    raise ConfigException(u"{key} does not translate to a list".format(key=key))
            return lst
        elif value is None:
            return None
        else:
            raise ConfigException(
                u"{key} has type '{type}' rather than 'list'".format(key=key, type=type(value).__name__))

    def get_config(self, key, default=UndefinedKey):
        """Return tree config representation of value found at key

        :param key: key to use (dot separated). E.g., a.b.c
        :type key: basestring
        :param default: default value if key not found
        :type default: config
        :return: config value
        :type return: ConfigTree
        """
        value = self.get(key, default)
        if isinstance(value, dict):
            return value
        elif value is None:
            return None
        else:
            raise ConfigException(
                u"{key} has type '{type}' rather than 'config'".format(key=key, type=type(value).__name__))

    def __getitem__(self, item):
        val = self.get(item)
        if val is UndefinedKey:
            raise KeyError(item)
        return val

    try:
        from collections import _OrderedDictItemsView
    except ImportError:  # pragma: nocover
        pass
    else:
        def items(self):  # pragma: nocover
            return self._OrderedDictItemsView(self)

    def __getattr__(self, item):
        val = self.get(item, NonExistentKey)
        if val is NonExistentKey:
            return super(ConfigTree, self).__getattr__(item)
        return val

    def __contains__(self, item):
        return self._get(self.parse_key(item), default=NoneValue) is not NoneValue

    def with_fallback(self, config, resolve=True):
        """
        return a new config with fallback on config
        :param config: config or filename of the config to fallback on
        :param resolve: resolve substitutions
        :return: new config with fallback on config
        """
        if isinstance(config, ConfigTree):
            result = ConfigTree.merge_configs(copy.deepcopy(config), copy.deepcopy(self))
        else:
            from . import ConfigFactory
            result = ConfigTree.merge_configs(ConfigFactory.parse_file(config, resolve=False), copy.deepcopy(self))

        if resolve:
            from . import ConfigParser
            ConfigParser.resolve_substitutions(result)
        return result

    def as_plain_ordered_dict(self):
        """return a deep copy of this config as a plain OrderedDict

        The config tree should be fully resolved.

        This is useful to get an object with no special semantics such as path expansion for the keys.
        In particular this means that keys that contain dots are not surrounded with '"' in the plain OrderedDict.

        :return: this config as an OrderedDict
        :type return: OrderedDict
        """
        def plain_value(v):
            if isinstance(v, list):
                return [plain_value(e) for e in v]
            elif isinstance(v, ConfigTree):
                return v.as_plain_ordered_dict()
            else:
                if isinstance(v, ConfigValues):
                    raise ConfigException("The config tree contains unresolved elements")
                return v

        return OrderedDict((key.strip('"') if isinstance(key, (unicode, basestring)) else key, plain_value(value))
                           for key, value in self.items())


class ConfigList(list):
    def __init__(self, iterable=[]):
        new_list = list(iterable)
        super(ConfigList, self).__init__(new_list)
        for index, value in enumerate(new_list):
            if isinstance(value, ConfigValues):
                value.parent = self
                value.key = index


class ConfigInclude(object):
    def __init__(self, tokens):
        self.tokens = tokens


class ConfigValues(object):
    def __init__(self, tokens, instring, loc):
        self.tokens = tokens
        self.parent = None
        self.key = None
        self._instring = instring
        self._loc = loc
        self.overriden_value = None
        self.recompute()

    def recompute(self):
        for index, token in enumerate(self.tokens):
            if isinstance(token, ConfigSubstitution):
                token.parent = self
                token.index = index

        # no value return empty string
        if len(self.tokens) == 0:
            self.tokens = ['']

        # if the last token is an unquoted string then right strip it
        if isinstance(self.tokens[-1], ConfigUnquotedString):
            # rstrip only whitespaces, not \n\r because they would have been used escaped
            self.tokens[-1] = self.tokens[-1].rstrip(' \t')

    def has_substitution(self):
        return len(self.get_substitutions()) > 0

    def get_substitutions(self):
        lst = []
        node = self
        while node:
            lst = [token for token in node.tokens if isinstance(token, ConfigSubstitution)] + lst
            if hasattr(node, 'overriden_value'):
                node = node.overriden_value
                if not isinstance(node, ConfigValues):
                    break
            else:
                break
        return lst

    def transform(self):
        def determine_type(token):
            return ConfigTree if isinstance(token, ConfigTree) else ConfigList if isinstance(token, list) else str

        def format_str(v, last=False):
            if isinstance(v, ConfigQuotedString):
                return v.value + ('' if last else v.ws)
            else:
                return '' if v is None else unicode(v)

        if self.has_substitution():
            return self

        # remove None tokens
        tokens = [token for token in self.tokens if token is not None]

        if not tokens:
            return None

        # check if all tokens are compatible
        first_tok_type = determine_type(tokens[0])
        for index, token in enumerate(tokens[1:]):
            tok_type = determine_type(token)
            if first_tok_type is not tok_type:
                raise ConfigWrongTypeException(
                    "Token '{token}' of type {tok_type} (index {index}) must be of type {req_tok_type} "
                    "(line: {line}, col: {col})".format(
                        token=token,
                        index=index + 1,
                        tok_type=tok_type.__name__,
                        req_tok_type=first_tok_type.__name__,
                        line=lineno(self._loc, self._instring),
                        col=col(self._loc, self._instring)))

        if first_tok_type is ConfigTree:
            child = []
            if hasattr(self, 'overriden_value'):
                node = self.overriden_value
                while node:
                    if isinstance(node, ConfigValues):
                        value = node.transform()
                        if isinstance(value, ConfigTree):
                            child.append(value)
                        else:
                            break
                    elif isinstance(node, ConfigTree):
                        child.append(node)
                    else:
                        break
                    if hasattr(node, 'overriden_value'):
                        node = node.overriden_value
                    else:
                        break

            result = ConfigTree()
            for conf in reversed(child):
                ConfigTree.merge_configs(result, conf, copy_trees=True)
            for token in tokens:
                ConfigTree.merge_configs(result, token, copy_trees=True)
            return result
        elif first_tok_type is ConfigList:
            result = []
            main_index = 0
            for sublist in tokens:
                sublist_result = ConfigList()
                for token in sublist:
                    if isinstance(token, ConfigValues):
                        token.parent = result
                        token.key = main_index
                    main_index += 1
                    sublist_result.append(token)
                result.extend(sublist_result)
            return result
        else:
            if len(tokens) == 1:
                if isinstance(tokens[0], ConfigQuotedString):
                    return tokens[0].value
                return tokens[0]
            else:
                return ''.join(format_str(token) for token in tokens[:-1]) + format_str(tokens[-1], True)

    def put(self, index, value):
        self.tokens[index] = value

    def __repr__(self):  # pragma: no cover
        return '[ConfigValues: ' + ','.join(str(o) for o in self.tokens) + ']'


class ConfigSubstitution(object):
    def __init__(self, variable, optional, ws, instring, loc):
        self.variable = variable
        self.optional = optional
        self.ws = ws
        self.index = None
        self.parent = None
        self.instring = instring
        self.loc = loc

    def __repr__(self):  # pragma: no cover
        return '[ConfigSubstitution: ' + self.variable + ']'


class ConfigUnquotedString(unicode):
    def __new__(cls, value):
        return super(ConfigUnquotedString, cls).__new__(cls, value)


class ConfigQuotedString(object):
    def __init__(self, value, ws, instring, loc):
        self.value = value
        self.ws = ws
        self.instring = instring
        self.loc = loc

    def __repr__(self):  # pragma: no cover
        return '[ConfigQuotedString: ' + self.value + ']'


# clearml/utilities/pyhocon/converter.py

import json
import re
import sys

from . import ConfigFactory
from .config_tree import ConfigQuotedString
from .config_tree import ConfigSubstitution
from .config_tree import ConfigTree
from .config_tree import ConfigValues
from .config_tree import NoneValue


try:
    basestring
except NameError:
    basestring = str
    unicode = str


class HOCONConverter(object):
    _number_re = r'[+-]?(\d*\.\d+|\d+(\.\d+)?)([eE][+\-]?\d+)?(?=$|[ \t]*([\$\}\],#\n\r]|//))'
    _number_re_matcher = re.compile(_number_re)

    @classmethod
    def to_json(cls, config, compact=False, indent=2, level=0):
        """Convert HOCON input into a JSON output

        :return: JSON string representation
        :type return: basestring
        """
        lines = ""
        if isinstance(config, ConfigTree):
            if len(config) == 0:
                lines += '{}'
            else:
                lines += '{\n'
                bet_lines = []
                for key, item in config.items():
                    bet_lines.append('{indent}"{key}": {value}'.format(
                        indent=''.rjust((level + 1) * indent, ' '),
                        key=key.strip('"'),  # for dotted keys enclosed with "" to not be interpreted as nested key
                        value=cls.to_json(item, compact, indent, level + 1))
                    )
                lines += ',\n'.join(bet_lines)
                lines += '\n{indent}}}'.format(indent=''.rjust(level * indent, ' '))
        elif isinstance(config, list):
            if len(config) == 0:
                lines += '[]'
            else:
                lines += '[\n'
                bet_lines = []
                for item in config:
                    bet_lines.append('{indent}{value}'.format(
                        indent=''.rjust((level + 1) * indent, ' '),
                        value=cls.to_json(item, compact, indent, level + 1))
                    )
                lines += ',\n'.join(bet_lines)
                lines += '\n{indent}]'.format(indent=''.rjust(level * indent, ' '))
        elif isinstance(config, basestring):
            lines = json.dumps(config)
        elif config is None or isinstance(config, NoneValue):
            lines = 'null'
        elif config is True:
            lines = 'true'
        elif config is False:
            lines = 'false'
        else:
            lines = str(config)
        return lines

    @staticmethod
    def _auto_indent(lines, section):
        # noinspection PyBroadException
        try:
            indent = len(lines) - lines.rindex('\n')
        except Exception:
            indent = len(lines)
        # noinspection PyBroadException
        try:
            section_indent = section.index('\n')
        except Exception:
            section_indent = len(section)
        if section_indent < 3:
            return lines + section

        indent = '\n' + ''.rjust(indent, ' ')
        return lines + indent.join([sec.strip() for sec in section.split('\n')])
        # indent = ''.rjust(indent, ' ')
        # return lines + section.replace('\n', '\n'+indent)

    @classmethod
    def to_hocon(cls, config, compact=False, indent=2, level=0):
        """Convert HOCON input into a HOCON output

        :return: JSON string representation
        :type return: basestring
        """
        lines = ""
        if isinstance(config, ConfigTree):
            if len(config) == 0:
                lines += '{}'
            else:
                if level > 0:  # don't display { at root level
                    lines += '{\n'
                bet_lines = []

                for key, item in config.items():
                    if compact:
                        full_key = key
                        while isinstance(item, ConfigTree) and len(item) == 1:
                            key, item = next(iter(item.items()))
                            full_key += '.' + key
                    else:
                        full_key = key

                    if isinstance(full_key, float) or \
                            (isinstance(full_key, (basestring, unicode)) and cls._number_re_matcher.match(full_key)):
                        # if key can be casted to float, and it is a string, make sure we quote it
                        full_key = '\"{}\"'.format(full_key)

                    bet_line = ('{indent}{key}{assign_sign} '.format(
                        indent=''.rjust(level * indent, ' '),
                        key=full_key,
                        assign_sign='' if isinstance(item, dict) else ' =',)
                    )
                    value_line = cls.to_hocon(item, compact, indent, level + 1)
                    if isinstance(item, (list, tuple)):
                        bet_lines.append(cls._auto_indent(bet_line, value_line))
                    else:
                        bet_lines.append(bet_line + value_line)
                lines += '\n'.join(bet_lines)

                if level > 0:  # don't display { at root level
                    lines += '\n{indent}}}'.format(indent=''.rjust((level - 1) * indent, ' '))
        elif isinstance(config, (list, tuple)):
            if len(config) == 0:
                lines += '[]'
            else:
                # lines += '[\n'
                lines += '['
                bet_lines = []
                base_len = len(lines)
                skip_comma = False
                for i, item in enumerate(config):
                    if 0 < i and not skip_comma:
                        # if not isinstance(item, (str, int, float)):
                        #     lines += ',\n{indent}'.format(indent=''.rjust(level * indent, ' '))
                        # else:
                        #     lines += ', '
                        lines += ', '

                    skip_comma = False
                    new_line = cls.to_hocon(item, compact, indent, level + 1)
                    lines += new_line
                    if '\n' in new_line or len(lines) - base_len > 80:
                        if i < len(config) - 1:
                            lines += ',\n{indent}'.format(indent=''.rjust(level * indent, ' '))
                        base_len = len(lines)
                        skip_comma = True
                    # bet_lines.append('{value}'.format(value=cls.to_hocon(item, compact, indent, level + 1)))

                # lines += '\n'.join(bet_lines)
                # lines += ', '.join(bet_lines)

                # lines += '\n{indent}]'.format(indent=''.rjust((level - 1) * indent, ' '))
                lines += ']'
        elif isinstance(config, basestring):
            if '\n' in config and len(config) > 1:
                lines = '"""{value}"""'.format(value=config)  # multilines
            else:
                lines = '"{value}"'.format(value=cls.__escape_string(config))
        elif isinstance(config, ConfigValues):
            lines = ''.join(cls.to_hocon(o, compact, indent, level) for o in config.tokens)
        elif isinstance(config, ConfigSubstitution):
            lines = '${'
            if config.optional:
                lines += '?'
            lines += config.variable + '}' + config.ws
        elif isinstance(config, ConfigQuotedString):
            if '\n' in config.value and len(config.value) > 1:
                lines = '"""{value}"""'.format(value=config.value)  # multilines
            else:
                lines = '"{value}"'.format(value=cls.__escape_string(config.value))
        elif config is None or isinstance(config, NoneValue):
            lines = 'null'
        elif config is True:
            lines = 'true'
        elif config is False:
            lines = 'false'
        else:
            lines = str(config)
        return lines

    @classmethod
    def to_yaml(cls, config, compact=False, indent=2, level=0):
        """Convert HOCON input into a YAML output

        :return: YAML string representation
        :type return: basestring
        """
        lines = ""
        if isinstance(config, ConfigTree):
            if len(config) > 0:
                if level > 0:
                    lines += '\n'
                bet_lines = []
                for key, item in config.items():
                    bet_lines.append('{indent}{key}: {value}'.format(
                        indent=''.rjust(level * indent, ' '),
                        key=key.strip('"'),  # for dotted keys enclosed with "" to not be interpreted as nested key,
                        value=cls.to_yaml(item, compact, indent, level + 1))
                    )
                lines += '\n'.join(bet_lines)
        elif isinstance(config, list):
            config_list = [line for line in config if line is not None]
            if len(config_list) == 0:
                lines += '[]'
            else:
                lines += '\n'
                bet_lines = []
                for item in config_list:
                    bet_lines.append('{indent}- {value}'.format(indent=''.rjust(level * indent, ' '),
                                                                value=cls.to_yaml(item, compact, indent, level + 1)))
                lines += '\n'.join(bet_lines)
        elif isinstance(config, basestring):
            # if it contains a \n then it's multiline
            lines = config.split('\n')
            if len(lines) == 1:
                lines = config
            else:
                lines = '|\n' + '\n'.join([line.rjust(level * indent, ' ') for line in lines])
        elif config is None or isinstance(config, NoneValue):
            lines = 'null'
        elif config is True:
            lines = 'true'
        elif config is False:
            lines = 'false'
        else:
            lines = str(config)
        return lines

    @classmethod
    def to_properties(cls, config, compact=False, indent=2, key_stack=[]):
        """Convert HOCON input into a .properties output

        :return: .properties string representation
        :type return: basestring
        :return:
        """

        def escape_value(value):
            return value.replace('=', '\\=').replace('!', '\\!').replace('#', '\\#').replace('\n', '\\\n')

        stripped_key_stack = [key.strip('"') for key in key_stack]
        lines = []
        if isinstance(config, ConfigTree):
            for key, item in config.items():
                if item is not None:
                    lines.append(cls.to_properties(item, compact, indent, stripped_key_stack + [key]))
        elif isinstance(config, list):
            for index, item in enumerate(config):
                if item is not None:
                    lines.append(cls.to_properties(item, compact, indent, stripped_key_stack + [str(index)]))
        elif isinstance(config, basestring):
            lines.append('.'.join(stripped_key_stack) + ' = ' + escape_value(config))
        elif config is True:
            lines.append('.'.join(stripped_key_stack) + ' = true')
        elif config is False:
            lines.append('.'.join(stripped_key_stack) + ' = false')
        elif config is None or isinstance(config, NoneValue):
            pass
        else:
            lines.append('.'.join(stripped_key_stack) + ' = ' + str(config))
        return '\n'.join([line for line in lines if len(line) > 0])

    @classmethod
    def convert(cls, config, output_format='json', indent=2, compact=False):
        converters = {
            'json': cls.to_json,
            'properties': cls.to_properties,
            'yaml': cls.to_yaml,
            'hocon': cls.to_hocon,
        }

        if output_format in converters:
            return converters[output_format](config, compact, indent)
        else:
            raise Exception("Invalid format '{format}'. Format must be 'json', 'properties', 'yaml' or 'hocon'".format(
                format=output_format))

    @classmethod
    def convert_from_file(cls, input_file=None, output_file=None, output_format='json', indent=2, compact=False):
        """Convert to json, properties or yaml

        :param input_file: input file, if not specified stdin
        :param output_file: output file, if not specified stdout
        :param output_format: json, properties or yaml
        :return: json, properties or yaml string representation
        """

        if input_file is None:
            content = sys.stdin.read()
            config = ConfigFactory.parse_string(content)
        else:
            config = ConfigFactory.parse_file(input_file)

        res = cls.convert(config, output_format, indent, compact)
        if output_file is None:
            print(res)
        else:
            with open(output_file, "w") as fd:
                fd.write(res)

    @classmethod
    def __escape_match(cls, match):
        char = match.group(0)
        return {
            '\b': r'\b',
            '\t': r'\t',
            '\n': r'\n',
            '\f': r'\f',
            '\r': r'\r',
            '"': r'\"',
            '\\': r'\\',
        }.get(char) or (r'\u%04x' % ord(char))

    @classmethod
    def __escape_string(cls, string):
        return re.sub(r'[\x00-\x1F"\\]', cls.__escape_match, string)


# clearml/utilities/pyhocon/exceptions.py

class ConfigException(Exception):

    def __init__(self, message, ex=None):
        super(ConfigException, self).__init__(message)
        self._exception = ex


class ConfigMissingException(ConfigException, KeyError):
    pass


class ConfigSubstitutionException(ConfigException):
    pass


class ConfigWrongTypeException(ConfigException):
    pass


# clearml/utilities/wizard/user_input.py

from typing import Optional


def get_input(
    key,  # type: str
    description="",  # type: str
    question="Enter",  # type: str
    required=False,  # type: bool
    default=None,  # type: Optional[str]
    new_line=False,  # type: bool
):
    # type: (...) -> Optional[str]
    if new_line:
        print()
    while True:
        value = input("{} {} {}: ".format(question, key, description))
        if not value.strip() and required:
            print("{} is required".format(key))
        elif not (value.strip() or required):
            return default
        else:
            return value


def input_int(
    key,  # type: str
    description="",  # type: str
    required=False,  # type: bool
    default=None,  # type: Optional[int]
    new_line=False,  # type: bool
):
    # type: (...) -> Optional[int]
    while True:
        try:
            value = int(
                get_input(
                    key,
                    description,
                    required=required,
                    default=default,
                    new_line=new_line,
                )
            )
            return value
        except ValueError:
            print(
                "Invalid input: {} should be a number. Please enter an integer".format(
                    key
                )
            )


def input_bool(question, default=False):
    # type: (str, bool) -> bool
    """
    :param question: string to display
    :param default: default boolean value
    :return: return True if response is 'y'/'yes' 't'/'true' in input.lower()
    """
    while True:
        try:
            response = input("{}: ".format(question)).lower()
            if not response:
                return default
            if response.startswith("y") or response.startswith("t"):
                return True
            if response.startswith("n") or response.startswith("f"):
                return False
            raise ValueError()
        except ValueError:
            print("Invalid input: please enter 'yes' or 'no'")


def input_list(
    key,  # type: str
    description="",  # type: str
    question="Enter",  # type: str
    required=False,  # type: bool
    default=None,  # type: Optional[str]
    new_line=False,  # type: bool
):
    res = get_input(key, description, question, required, default, new_line)
    if not res:
        return None

    res_list = [res]
    while input_bool("\nDefine another {}? [y/N]".format(key)):
        response = get_input(
                    key=key,
                    description=description,
                    question=question,
                    required=False,
                    default=default,
                    new_line=new_line,
                )
        if response:
            res_list.append(response)
    return res_list


def multiline_input(description=""):
    print("{} \nNote: two consecutive empty lines would terminate the input : ".format(description))
    lines = []
    empty_lines = 0
    while empty_lines < 2:
        line = input()
        lines.append(line)
        empty_lines = 0 if line else empty_lines + 1
    res = "\n".join(lines[:-1])
    return res, len(res.splitlines())


# clearml/__init__.py

""" ClearML open SDK """
from six import PY2

from .version import __version__
from .task import Task
from .model import InputModel, OutputModel, Model
from .logger import Logger
from .storage import StorageManager
from .errors import UsageError
from .datasets import Dataset

TaskTypes = Task.TaskTypes

if not PY2:
    from .automation.controller import PipelineController, PipelineDecorator  # noqa: F401

    __all__ = [
        "__version__",
        "Task",
        "TaskTypes",
        "InputModel",
        "OutputModel",
        "Model",
        "Logger",
        "StorageManager",
        "UsageError",
        "Dataset",
        "PipelineController",
        "PipelineDecorator",
    ]
else:
    __all__ = [
        "__version__",
        "Task",
        "TaskTypes",
        "InputModel",
        "OutputModel",
        "Model",
        "Logger",
        "StorageManager",
        "UsageError",
        "Dataset",
    ]


# clearml/errors.py

class UsageError(RuntimeError):
    """ An exception raised for illegal usage of clearml objects"""
    pass


# clearml/logger.py

import logging
import math
import warnings
from typing import Any, Sequence, Union, List, Optional, Tuple, Dict, TYPE_CHECKING

import numpy as np
import six
from PIL import Image
from pathlib2 import Path

from .debugging.log import LoggerRoot

try:
    import pandas as pd
except ImportError:
    pd = None

from .backend_interface.logger import StdStreamPatch
from .backend_interface.task import Task as _Task
from .backend_interface.task.log import TaskHandler
from .backend_interface.util import mutually_exclusive
from .backend_interface.metrics.events import UploadEvent, MetricsEventAdapter
from .config import running_remotely, get_cache_dir, config, DEBUG_SIMULATE_REMOTE_TASK, deferred_config
from .errors import UsageError
from .storage.helper import StorageHelper
from .utilities.plotly_reporter import SeriesInfo

# Make sure that DeprecationWarning within this package always gets printed
warnings.filterwarnings('always', category=DeprecationWarning, module=__name__)


if TYPE_CHECKING:
    from matplotlib.figure import Figure as MatplotlibFigure  # noqa
    from matplotlib import pyplot  # noqa


class Logger(object):
    """
    The ``Logger`` class is the ClearML console log and metric statistics interface, and contains methods for explicit
    reporting.

    Explicit reporting extends ClearML automagical capturing of inputs and output. Explicit reporting
    methods include scalar plots, line plots, histograms, confusion matrices, 2D and 3D scatter
    diagrams, text logging, tables, and image uploading and reporting.

    In the **ClearML Web-App (UI)**, ``Logger`` output appears in the **RESULTS** tab, **CONSOLE**, **SCALARS**,
    **PLOTS**, and **DEBUG SAMPLES** sub-tabs. When you compare experiments, ``Logger`` output appears in the
    comparisons.

    .. warning::

       Do not construct Logger objects directly.

    You must get a Logger object before calling any of the other ``Logger`` class methods by calling
    :meth:`.Task.get_logger` or :meth:`Logger.current_logger`.


    """
    SeriesInfo = SeriesInfo
    _tensorboard_logging_auto_group_scalars = False
    _tensorboard_single_series_per_graph = deferred_config('metrics.tensorboard_single_series_per_graph', False)

    def __init__(self, private_task, connect_stdout=True, connect_stderr=True, connect_logging=False):
        """
        .. warning::
            **Do not construct Logger manually!**
            Please use :meth:`Logger.get_current`
        """
        assert isinstance(private_task, _Task), \
            'Logger object cannot be instantiated externally, use Logger.current_logger()'
        super(Logger, self).__init__()
        self._task = private_task
        self._default_upload_destination = None
        self._flusher = None
        self._report_worker = None
        self._graph_titles = {}
        self._tensorboard_series_force_prefix = None
        self._task_handler = TaskHandler(task=self._task, capacity=100) \
            if private_task.is_main_task() or (connect_stdout or connect_stderr or connect_logging) else None
        self._connect_std_streams = connect_stdout or connect_stderr
        self._connect_logging = connect_logging
        self._default_max_sample_history = None

        # Make sure urllib is never in debug/info,
        disable_urllib3_info = config.get('log.disable_urllib3_info', True)
        if disable_urllib3_info and logging.getLogger('urllib3').isEnabledFor(logging.INFO):
            logging.getLogger('urllib3').setLevel(logging.WARNING)

        StdStreamPatch.patch_std_streams(self, connect_stdout=connect_stdout, connect_stderr=connect_stderr)

        if self._connect_logging:
            StdStreamPatch.patch_logging_formatter(self)
        elif not self._connect_std_streams:
            # make sure that at least the main clearml logger is connect
            base_logger = LoggerRoot.get_base_logger()
            if base_logger and base_logger.handlers:
                StdStreamPatch.patch_logging_formatter(self, base_logger.handlers[0])

    @classmethod
    def current_logger(cls):
        # type: () -> Logger
        """
        Get the Logger object for the main execution Task, the current running Task, if one exists. If no Logger object
        exists, this method creates one and returns it. Therefore, you can call this method from anywhere
        in the code.

        .. code-block:: py

           logger = Logger.current_logger()

        :return: The Logger object (a singleton) for the current running Task.
        """
        from .task import Task
        task = Task.current_task()
        if not task:
            return None
        return task.get_logger()

    def report_text(self, msg, level=logging.INFO, print_console=True, *args, **_):
        # type: (str, int, bool, Any, Any) -> None
        """
        For explicit reporting, print text to the log. Optionally, print a log level and print to the console.

        For example:

        .. code-block:: py

           logger.report_text('log some text', level=logging.DEBUG, print_console=False)

        You can view the reported text in the **ClearML Web-App (UI)**, **RESULTS** tab, **CONSOLE** sub-tab.

        :param str msg: The text to log.
        :param int level: The log level from the Python ``logging`` package. The default value is ``logging.INFO``.
        :param bool print_console: In addition to the log, print to the console

            The values are:

            - ``True`` - Print to the console. (default)
            - ``False`` - Do not print to the console.
        """
        force_send = not print_console and self._parse_level(level) >= logging.WARNING
        return self._console(msg, level, not print_console, force_send=force_send, *args, **_)

    def report_scalar(self, title, series, value, iteration):
        # type: (str, str, float, int) -> None
        """
        For explicit reporting, plot a scalar series.

        For example, plot a scalar series:

        .. code-block:: py

           logger = Logger.current_logger()
           scalar_series = [random.randint(0,10) for i in range(10)]
           for iteration in range(10):
               logger.report_scalar(
                   title='scalar metrics', series='series', value=scalar_series[iteration], iteration=iteration
               )

        You can view the scalar plots in the **ClearML Web-App (UI)**, **RESULTS** tab, **SCALARS** sub-tab.

        :param str title: The title (metric) of the plot. Plot more than one scalar series on the same plot by using
            the same ``title`` for each call to this method.
        :param str series: The series name (variant) of the reported scalar.
        :param float value: The value to plot per iteration.
        :param int iteration: The reported iteration / step (x-axis of the reported time series)
        """

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_scalar(title=title, series=series, value=float(value), iter=iteration)

    def report_single_value(self, name, value):
        # type: (str, float) -> None
        """
        Reports a single value metric (for example, total experiment accuracy or mAP)
        You can view the metrics in the **ClearML Web-App (UI)**, **RESULTS** tab, **SCALARS** sub-tab.

        :param name: Metric's name
        :param value: Metric's value
        """
        return self.report_scalar(title="Summary", series=name, value=value, iteration=-2**31)

    def report_vector(
            self,
            title,  # type: str
            series,  # type: str
            values,  # type: Sequence[Union[int, float]]
            iteration=None,  # type: Optional[int]
            labels=None,  # type: Optional[List[str]]
            xlabels=None,  # type: Optional[List[str]]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            mode=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a vector as (default stacked) histogram.

        For example:

        .. code-block:: py

           vector_series = np.random.randint(10, size=10).reshape(2,5)
           logger.report_vector(title='vector example', series='vector series', values=vector_series, iteration=0,
                labels=['A','B'], xaxis='X axis label', yaxis='Y axis label')

        You can view the vectors plots in the **ClearML Web-App (UI)**, **RESULTS** tab, **PLOTS** sub-tab.

        :param title: The title (metric) of the plot.
        :param series: The series name (variant) of the reported histogram.
        :param values: The series values. A list of floats, or an N-dimensional Numpy array containing
            data for each histogram bar.
        :param iteration: The reported iteration / step. Each ``iteration`` creates another plot.
        :param labels: Labels for each bar group, creating a plot legend labeling each series. (Optional)
        :param xlabels: Labels per entry in each bucket in the histogram (vector), creating a set of labels
            for each histogram bar on the x-axis. (Optional)
        :param xaxis: The x-axis title. (Optional)
        :param yaxis: The y-axis title. (Optional)
        :param mode: Multiple histograms mode, stack / group / relative. Default is 'group'.
        :param extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/bar/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """
        self._touch_title_series(title, series)
        return self.report_histogram(title, series, values, iteration or 0, labels=labels, xlabels=xlabels,
                                     xaxis=xaxis, yaxis=yaxis, mode=mode, extra_layout=extra_layout)

    def report_histogram(
            self,
            title,  # type: str
            series,  # type: str
            values,  # type: Sequence[Union[int, float]]
            iteration=None,  # type: Optional[int]
            labels=None,  # type: Optional[List[str]]
            xlabels=None,  # type: Optional[List[str]]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            mode=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a (default grouped) histogram.
        Notice this function will not calculate the histogram,
        it assumes the histogram was already calculated in `values`

        For example:

        .. code-block:: py

           vector_series = np.random.randint(10, size=10).reshape(2,5)
           logger.report_histogram(title='histogram example', series='histogram series',
                values=vector_series, iteration=0, labels=['A','B'], xaxis='X axis label', yaxis='Y axis label')

        You can view the reported histograms in the **ClearML Web-App (UI)**, **RESULTS** tab, **PLOTS** sub-tab.

        :param title: The title (metric) of the plot.
        :param series: The series name (variant) of the reported histogram.
        :param values: The series values. A list of floats, or an N-dimensional Numpy array containing
            data for each histogram bar.
        :param iteration: The reported iteration / step. Each ``iteration`` creates another plot.
        :param labels: Labels for each bar group, creating a plot legend labeling each series. (Optional)
        :param xlabels: Labels per entry in each bucket in the histogram (vector), creating a set of labels
            for each histogram bar on the x-axis. (Optional)
        :param xaxis: The x-axis title. (Optional)
        :param yaxis: The y-axis title. (Optional)
        :param mode: Multiple histograms mode, stack / group / relative. Default is 'group'.
        :param extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/bar/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        if not isinstance(values, np.ndarray):
            values = np.array(values)

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_histogram(
            title=title,
            series=series,
            histogram=values,
            iter=iteration or 0,
            labels=labels,
            xlabels=xlabels,
            xtitle=xaxis,
            ytitle=yaxis,
            mode=mode or 'group',
            layout_config=extra_layout,
        )

    def report_table(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            table_plot=None,  # type: Optional[pd.DataFrame, Sequence[Sequence]]
            csv=None,  # type: Optional[str]
            url=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit report, report a table plot.

        One and only one of the following parameters must be provided.

        - ``table_plot`` - Pandas DataFrame or Table as list of rows (list)
        - ``csv`` - CSV file
        - ``url`` - URL to CSV file

        For example:

        .. code-block:: py

           df = pd.DataFrame({'num_legs': [2, 4, 8, 0],
                   'num_wings': [2, 0, 0, 0],
                   'num_specimen_seen': [10, 2, 1, 8]},
                   index=['falcon', 'dog', 'spider', 'fish'])

           logger.report_table(title='table example',series='pandas DataFrame',iteration=0,table_plot=df)

        You can view the reported tables in the **ClearML Web-App (UI)**, **RESULTS** tab, **PLOTS** sub-tab.

        :param title: The title (metric) of the table.
        :param series: The series name (variant) of the reported table.
        :param iteration: The reported iteration / step.
        :param table_plot: The output table plot object
        :param csv: path to local csv file
        :param url: A URL to the location of csv file.
        :param extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/table/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """
        mutually_exclusive(
            UsageError, _check_none=True,
            table_plot=table_plot, csv=csv, url=url
        )
        table = table_plot
        if url or csv:
            if not pd:
                raise UsageError(
                    "pandas is required in order to support reporting tables using CSV or a URL, "
                    "please install the pandas python package"
                )
            if url:
                table = pd.read_csv(url, index_col=[0])
            elif csv:
                table = pd.read_csv(csv, index_col=[0])

        def replace(dst, *srcs):
            for src in srcs:
                reporter_table.replace(src, dst, inplace=True)

        if isinstance(table, (list, tuple)):
            reporter_table = table
        else:
            reporter_table = table.fillna(str(np.nan))
            replace("NaN", np.nan, math.nan if six.PY3 else float("nan"))
            replace("Inf", np.inf, math.inf if six.PY3 else float("inf"))
            replace("-Inf", -np.inf, np.NINF, -math.inf if six.PY3 else -float("inf"))
        # noinspection PyProtectedMember
        return self._task._reporter.report_table(
            title=title,
            series=series,
            table=reporter_table,
            iteration=iteration or 0,
            layout_config=extra_layout,
        )

    def report_line_plot(
            self,
            title,  # type: str
            series,  # type: Sequence[SeriesInfo]
            xaxis,  # type: str
            yaxis,  # type: str
            mode='lines',  # type: str
            iteration=None,  # type: Optional[int]
            reverse_xaxis=False,  # type: bool
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot one or more series as lines.

        :param str title: The title (metric) of the plot.
        :param list series: All the series data, one list element for each line in the plot.
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param str mode: The type of line plot.

            The values are:

            - ``lines`` (default)
            - ``markers``
            - ``lines+markers``

        :param bool reverse_xaxis: Reverse the x-axis

            The values are:

            - ``True`` - The x-axis is high to low  (reversed).
            - ``False`` - The x-axis is low to high  (not reversed). (default)

        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/scatter/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        # noinspection PyArgumentList
        series = [self.SeriesInfo(**s) if isinstance(s, dict) else s for s in series]

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series[0].name if series else '')
        # noinspection PyProtectedMember
        return self._task._reporter.report_line_plot(
            title=title,
            series=series,
            iter=iteration or 0,
            xtitle=xaxis,
            ytitle=yaxis,
            mode=mode,
            reverse_xaxis=reverse_xaxis,
            comment=comment,
            layout_config=extra_layout,
        )

    def report_scatter2d(
            self,
            title,  # type: str
            series,  # type: str
            scatter,  # type: Union[Sequence[Tuple[float, float]], np.ndarray]
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            labels=None,  # type: Optional[List[str]]
            mode='lines',  # type: str
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, report a 2d scatter plot.

        For example:

        .. code-block:: py

           scatter2d = np.hstack((np.atleast_2d(np.arange(0, 10)).T, np.random.randint(10, size=(10, 1))))
           logger.report_scatter2d(title="example_scatter", series="series", iteration=0, scatter=scatter2d,
                xaxis="title x", yaxis="title y")

        Plot multiple 2D scatter series on the same plot by passing the same ``title`` and ``iteration`` values
        to this method:

        .. code-block:: py

           scatter2d_1 = np.hstack((np.atleast_2d(np.arange(0, 10)).T, np.random.randint(10, size=(10, 1))))
           logger.report_scatter2d(title="example_scatter", series="series_1", iteration=1, scatter=scatter2d_1,
                xaxis="title x", yaxis="title y")

           scatter2d_2 = np.hstack((np.atleast_2d(np.arange(0, 10)).T, np.random.randint(10, size=(10, 1))))
           logger.report_scatter2d("example_scatter", "series_2", iteration=1, scatter=scatter2d_2,
                xaxis="title x", yaxis="title y")

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported scatter plot.
        :param list scatter: The scatter data. numpy.ndarray or list of (pairs of x,y) scatter:
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param list(str) labels: Labels per point in the data assigned to the ``scatter`` parameter. The labels must be
            in the same order as the data.
        :param str mode: The type of scatter plot.

            The values are:

            - ``lines``
            - ``markers``
            - ``lines+markers``

        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/scatter/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        if not isinstance(scatter, np.ndarray):
            if not isinstance(scatter, list):
                scatter = list(scatter)
            scatter = np.array(scatter)

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_2d_scatter(
            title=title,
            series=series,
            data=scatter,
            iter=iteration or 0,
            mode=mode,
            xtitle=xaxis,
            ytitle=yaxis,
            labels=labels,
            comment=comment,
            layout_config=extra_layout,
        )

    def report_scatter3d(
            self,
            title,  # type: str
            series,  # type: str
            scatter,  # type: Union[Sequence[Tuple[float, float, float]], np.ndarray]
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            zaxis=None,  # type: Optional[str]
            labels=None,  # type: Optional[List[str]]
            mode='markers',  # type: str
            fill=False,  # type: bool
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a 3d scatter graph (with markers).

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported scatter plot.
        :param Union[numpy.ndarray, list] scatter: The scatter data.
            list of (pairs of x,y,z), list of series [[(x1,y1,z1)...]], or numpy.ndarray
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param str zaxis: The z-axis title. (Optional)
        :param list(str) labels: Labels per point in the data assigned to the ``scatter`` parameter. The labels must be
            in the same order as the data.
        :param str mode: The type of scatter plot.

            The values are:

            - ``lines``
            - ``markers``
            - ``lines+markers``

        For example:

        .. code-block:: py

           scatter3d = np.random.randint(10, size=(10, 3))
           logger.report_scatter3d(title="example_scatter_3d", series="series_xyz", iteration=1, scatter=scatter3d,
                xaxis="title x", yaxis="title y", zaxis="title z")

        :param bool fill: Fill the area under the curve

            The values are:

            - ``True`` - Fill
            - ``False`` - Do not fill (default)

        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/scatter3d/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """
        # check if multiple series
        multi_series = (
            isinstance(scatter, list)
            and (
                isinstance(scatter[0], np.ndarray)
                or (
                    scatter[0]
                    and isinstance(scatter[0], list)
                    and isinstance(scatter[0][0], list)
                )
            )
        )

        if not multi_series:
            if not isinstance(scatter, np.ndarray):
                if not isinstance(scatter, list):
                    scatter = list(scatter)
                scatter = np.array(scatter)
            try:
                scatter = scatter.astype(np.float32)
            except ValueError:
                pass

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_3d_scatter(
            title=title,
            series=series,
            data=scatter,
            iter=iteration or 0,
            labels=labels,
            mode=mode,
            fill=fill,
            comment=comment,
            xtitle=xaxis,
            ytitle=yaxis,
            ztitle=zaxis,
            layout_config=extra_layout,
        )

    def report_confusion_matrix(
            self,
            title,  # type: str
            series,  # type: str
            matrix,  # type: np.ndarray
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            xlabels=None,  # type: Optional[List[str]]
            ylabels=None,  # type: Optional[List[str]]
            yaxis_reversed=False,  # type: bool
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a heat-map matrix.

        For example:

        .. code-block:: py

           confusion = np.random.randint(10, size=(10, 10))
           logger.report_confusion_matrix("example confusion matrix", "ignored", iteration=1, matrix=confusion,
                xaxis="title X", yaxis="title Y")

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported confusion matrix.
        :param numpy.ndarray matrix: A heat-map matrix (example: confusion matrix)
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param list(str) xlabels: Labels for each column of the matrix. (Optional)
        :param list(str) ylabels: Labels for each row of the matrix. (Optional)
        :param bool yaxis_reversed: If False 0,0 is at the bottom left corner. If True 0,0 is at the Top left corner
        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/heatmap/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        if not isinstance(matrix, np.ndarray):
            matrix = np.array(matrix)

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_value_matrix(
            title=title,
            series=series,
            data=matrix.astype(np.float32),
            iter=iteration or 0,
            xtitle=xaxis,
            ytitle=yaxis,
            xlabels=xlabels,
            ylabels=ylabels,
            yaxis_reversed=yaxis_reversed,
            comment=comment,
            layout_config=extra_layout,
        )

    def report_matrix(
            self,
            title,  # type: str
            series,  # type: str
            matrix,  # type: np.ndarray
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            xlabels=None,  # type: Optional[List[str]]
            ylabels=None,  # type: Optional[List[str]]
            yaxis_reversed=False,  # type: bool
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, plot a confusion matrix.

        .. note::
            This method is the same as :meth:`Logger.report_confusion_matrix`.

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported confusion matrix.
        :param numpy.ndarray matrix: A heat-map matrix (example: confusion matrix)
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param list(str) xlabels: Labels for each column of the matrix. (Optional)
        :param list(str) ylabels: Labels for each row of the matrix. (Optional)
        :param bool yaxis_reversed: If False 0,0 is at the bottom left corner. If True 0,0 is at the Top left corner
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/heatmap/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """
        self._touch_title_series(title, series)
        return self.report_confusion_matrix(title, series, matrix, iteration or 0,
                                            xaxis=xaxis, yaxis=yaxis, xlabels=xlabels, ylabels=ylabels,
                                            yaxis_reversed=yaxis_reversed,
                                            extra_layout=extra_layout)

    def report_surface(
            self,
            title,  # type: str
            series,  # type: str
            matrix,  # type: np.ndarray
            iteration=None,  # type: Optional[int]
            xaxis=None,  # type: Optional[str]
            yaxis=None,  # type: Optional[str]
            zaxis=None,  # type: Optional[str]
            xlabels=None,  # type: Optional[List[str]]
            ylabels=None,  # type: Optional[List[str]]
            camera=None,  # type: Optional[Sequence[float]]
            comment=None,  # type: Optional[str]
            extra_layout=None,  # type: Optional[dict]
    ):
        """
        For explicit reporting, report a 3d surface plot.

        .. note::
           This method plots the same data as :meth:`Logger.report_confusion_matrix`, but presents the
           data as a surface diagram not a confusion matrix.

        .. code-block:: py

           surface_matrix = np.random.randint(10, size=(10, 10))
           logger.report_surface("example surface", "series", iteration=0, matrix=surface_matrix,
                xaxis="title X", yaxis="title Y", zaxis="title Z")

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported surface.
        :param numpy.ndarray matrix: A heat-map matrix (example: confusion matrix)
        :param int iteration: The reported iteration / step.
        :param str xaxis: The x-axis title. (Optional)
        :param str yaxis: The y-axis title. (Optional)
        :param str zaxis: The z-axis title. (Optional)
        :param list(str) xlabels: Labels for each column of the matrix. (Optional)
        :param list(str) ylabels: Labels for each row of the matrix. (Optional)
        :param list(float) camera: X,Y,Z coordinates indicating the camera position. The default value is ``(1,1,1)``.
        :param str comment: A comment displayed with the plot, underneath the title.
        :param dict extra_layout: optional dictionary for layout configuration, passed directly to plotly
            See full details on the supported configuration: https://plotly.com/javascript/reference/surface/
            example: extra_layout={'xaxis': {'type': 'date', 'range': ['2020-01-01', '2020-01-31']}}
        """

        if not isinstance(matrix, np.ndarray):
            matrix = np.array(matrix)

        # if task was not started, we have to start it
        self._start_task_if_needed()
        self._touch_title_series(title, series)
        # noinspection PyProtectedMember
        return self._task._reporter.report_value_surface(
            title=title,
            series=series,
            data=matrix.astype(np.float32),
            iter=iteration or 0,
            xlabels=xlabels,
            ylabels=ylabels,
            xtitle=xaxis,
            ytitle=yaxis,
            ztitle=zaxis,
            camera=camera,
            comment=comment,
            layout_config=extra_layout,
        )

    def report_image(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            local_path=None,  # type: Optional[str]
            image=None,  # type: Optional[Union[np.ndarray, Image.Image]]
            matrix=None,  # type: Optional[np.ndarray]
            max_image_history=None,  # type: Optional[int]
            delete_after_upload=False,  # type: bool
            url=None  # type: Optional[str]
    ):
        """
        For explicit reporting, report an image and upload its contents.

        This method uploads the image to a preconfigured bucket (see :meth:`Logger.set_default_upload_destination`)
        describing the task ID, title, series and iteration.

        For example:

        .. code-block:: py

           matrix = np.eye(256, 256, dtype=np.uint8)*255
           matrix = np.concatenate((np.atleast_3d(matrix), np.zeros((256, 256, 2), dtype=np.uint8)), axis=2)
           logger.report_image("test case", "image color red", iteration=1, image=m)

           image_open = Image.open(os.path.join("<image_path>", "<image_filename>"))
           logger.report_image("test case", "image PIL", iteration=1, image=image_open)

        One and only one of the following parameters must be provided.

        - ``local_path``
        - ``url``
        - ``image``
        - ``matrix``

        :param title: The title (metric) of the image.
        :param series: The series name (variant) of the reported image.
        :param iteration: The reported iteration / step.
        :param local_path: A path to an image file.
        :param url: A URL for the location of a pre-uploaded image.
        :param image: Image data (RGB).
        :param matrix: Deperacted, Image data (RGB).

            .. note::
               The ``matrix`` parameter is deprecated. Use the ``image`` parameters.
        :param max_image_history: The maximum number of images to store per metric/variant combination.
            For an unlimited number, use a negative value. The default value is set in global configuration
            (default=``5``).
        :param delete_after_upload: After the upload, delete the local copy of the image

            The values are:

            - ``True`` - Delete after upload.
            - ``False`` - Do not delete after upload. (default)
        """
        mutually_exclusive(
            UsageError, _check_none=True,
            local_path=local_path or None, url=url or None, image=image, matrix=matrix
        )
        if matrix is not None:
            warnings.warn("'matrix' variable is deprecated; use 'image' instead.", DeprecationWarning)
        if image is None:
            image = matrix
        if image is not None and not isinstance(image, (np.ndarray, Image.Image)):
            raise ValueError("Supported 'image' types are: numpy.ndarray or PIL.Image")

        # if task was not started, we have to start it
        self._start_task_if_needed()

        self._touch_title_series(title, series)

        if url:
            # noinspection PyProtectedMember
            self._task._reporter.report_image(
                title=title,
                series=series,
                src=url,
                iter=iteration or 0,
            )

        else:
            upload_uri = self.get_default_upload_destination()
            if not upload_uri:
                upload_uri = Path(get_cache_dir()) / 'debug_images'
                upload_uri.mkdir(parents=True, exist_ok=True)
                # Verify that we can upload to this destination
                upload_uri = str(upload_uri)
                storage = StorageHelper.get(upload_uri)
                upload_uri = storage.verify_upload(folder_uri=upload_uri)

            if isinstance(image, Image.Image):
                image = np.array(image)  # noqa
            # noinspection PyProtectedMember
            self._task._reporter.report_image_and_upload(
                title=title,
                series=series,
                path=local_path,
                image=image,
                iter=iteration or 0,
                upload_uri=upload_uri,
                max_image_history=max_image_history if max_image_history is not None
                else self._default_max_sample_history,
                delete_after_upload=delete_after_upload,
            )

    def report_media(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            local_path=None,  # type: Optional[str]
            stream=None,  # type: Optional[Union[six.BytesIO, six.StringIO]]
            file_extension=None,  # type: Optional[str]
            max_history=None,  # type: Optional[int]
            delete_after_upload=False,  # type: bool
            url=None  # type: Optional[str]
    ):
        """
        Report media upload its contents, including images, audio, and video.

        Media is uploaded to a preconfigured bucket (see setup_upload()) with a key (filename)
        describing the task ID, title, series and iteration.

        One and only one of the following parameters must be provided

        - ``local_path``
        - ``stream``
        - ``url``

        If you use ``stream`` for a BytesIO stream to upload, ``file_extension`` must be provided.

        :param str title: The title (metric) of the media.
        :param str series: The series name (variant) of the reported media.
        :param int iteration: The reported iteration / step.
        :param str local_path: A path to an media file.
        :param stream: BytesIO stream to upload. If provided, ``file_extension`` must also be provided.
        :param str url: A URL to the location of a pre-uploaded media.
        :param file_extension: A file extension to use when ``stream`` is passed.
        :param int max_history: The maximum number of media files to store per metric/variant combination
            use negative value for unlimited. default is set in global configuration (default=5)
        :param bool delete_after_upload: After the file is uploaded, delete the local copy

            - ``True`` - Delete
            - ``False`` - Do not delete

        """
        mutually_exclusive(
            UsageError, _check_none=True,
            local_path=local_path or None, url=url or None, stream=stream,
        )
        if stream is not None and not file_extension:
            raise ValueError("No file extension provided for stream media upload")

        # if task was not started, we have to start it
        self._start_task_if_needed()

        self._touch_title_series(title, series)

        if url:
            # noinspection PyProtectedMember
            self._task._reporter.report_media(
                title=title,
                series=series,
                src=url,
                iter=iteration or 0,
            )

        else:
            upload_uri = self.get_default_upload_destination()
            if not upload_uri:
                upload_uri = Path(get_cache_dir()) / 'debug_images'
                upload_uri.mkdir(parents=True, exist_ok=True)
                # Verify that we can upload to this destination
                upload_uri = str(upload_uri)
                storage = StorageHelper.get(upload_uri)
                upload_uri = storage.verify_upload(folder_uri=upload_uri)
            # noinspection PyProtectedMember
            self._task._reporter.report_media_and_upload(
                title=title,
                series=series,
                path=local_path,
                stream=stream,
                iter=iteration or 0,
                upload_uri=upload_uri,
                max_history=max_history if max_history is not None else self._default_max_sample_history,
                delete_after_upload=delete_after_upload,
                file_extension=file_extension,
            )

    def report_plotly(
            self,
            title,  # type: str
            series,  # type: str
            figure,  # type: Union[Dict, "Figure"]  # noqa: F821
            iteration=None,  # type: Optional[int]
    ):
        """
        Report a ``Plotly`` figure (plot) directly

        ``Plotly`` figure can be a ``plotly.graph_objs._figure.Figure`` or a dictionary as defined by ``plotly.js``

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported plot.
        :param int iteration: The reported iteration / step.
        :param dict figure: A ``plotly`` Figure object or a ``poltly`` dictionary
        """
        # if task was not started, we have to start it
        self._start_task_if_needed()

        self._touch_title_series(title, series)

        plot = figure if isinstance(figure, dict) else figure.to_plotly_json()
        # noinspection PyBroadException
        try:
            plot['layout']['title'] = series
        except Exception:
            pass
        # noinspection PyProtectedMember
        self._task._reporter.report_plot(
            title=title,
            series=series,
            plot=plot,
            iter=iteration or 0,
        )

    def report_matplotlib_figure(
            self,
            title,  # type: str
            series,  # type: str
            figure,  # type: Union[MatplotlibFigure, pyplot]
            iteration=None,  # type: Optional[int]
            report_image=False,  # type: bool
            report_interactive=True,  # type: bool
    ):
        """
        Report a ``matplotlib`` figure / plot directly

        ``matplotlib.figure.Figure`` / ``matplotlib.pyplot``

        :param str title: The title (metric) of the plot.
        :param str series: The series name (variant) of the reported plot.
        :param int iteration: The reported iteration / step.
        :param MatplotlibFigure figure: A ``matplotlib`` Figure object
        :param report_image: Default False. If True the plot will be uploaded as a debug sample (png image),
            and will appear under the debug samples tab (instead of the Plots tab).
        :param report_interactive: If True (default) it will try to convert the matplotlib into interactive
            plot in the UI. If False the matplotlib is saved as is and will
            be non-interactive (with the exception of zooming in/out)
        """
        # if task was not started, we have to start it
        self._start_task_if_needed()

        # noinspection PyProtectedMember
        self._task._reporter.report_matplotlib(
            title=title,
            series=series,
            figure=figure,
            iter=iteration or 0,
            logger=self,
            force_save_as_image=False if report_interactive and not report_image
            else ('png' if report_image else True),
        )

    def set_default_upload_destination(self, uri):
        # type: (str) -> None
        """
        Set the destination storage URI (for example, S3, Google Cloud Storage, a file path) for uploading debug images.

        The images are uploaded separately. A link to each image is reported.

        .. note::
           Credentials for the destination storage are specified in the  ClearML configuration file,
           ``~/clearml.conf``.

        :param str uri: example: 's3://bucket/directory/' or 'file:///tmp/debug/'

        :return: True, if the destination scheme is supported (for example, ``s3://``, ``file://``, or ``gc://``).
            False, if not supported.

        """

        # Create the storage helper
        storage = StorageHelper.get(uri)

        # Verify that we can upload to this destination
        uri = storage.verify_upload(folder_uri=uri)

        self._default_upload_destination = uri

    def get_default_upload_destination(self):
        # type: () -> str
        """
        Get the destination storage URI (for example, S3, Google Cloud Storage, a file path) for uploading debug images
        (see :meth:`Logger.set_default_upload_destination`).

        :return: The default upload destination URI.

            For example: ``s3://bucket/directory/``, or ``file:///tmp/debug/``.
        """
        # noinspection PyProtectedMember
        return self._default_upload_destination or self._task._get_default_report_storage_uri()

    def flush(self, wait=False):
        # type: (bool) -> bool
        """
        Flush cached reports and console outputs to backend.

        :param wait: Wait for all outstanding uploads and events to be sent (default False)

        :return: True, if successfully flushed the cache. False, if failed.
        """
        self._flush_stdout_handler()
        if self._task:
            return self._task.flush(wait_for_uploads=wait)
        return False

    def get_flush_period(self):
        # type: () -> Optional[float]
        """
        Get the Logger flush period.

        :return: The logger flush period in seconds.
        """
        if self._flusher:
            return self._flusher.period
        return None

    def set_flush_period(self, period):
        # type: (float) -> None
        """
        Set the logger flush period.

        Deprecated - Use ``sdk.development.worker.report_period_sec`` to externally control the flush period.

        :param float period: The period to flush the logger in seconds. To set no periodic flush,
            specify ``None`` or ``0``.
        """
        pass

    def set_default_debug_sample_history(self, max_history):
        # type: (int) -> None
        """
        Set the default maximum debug sample history when reporting media/debug samples.
        Overrides the configuration file defaults.
        When reporting debug samples with the same title/series combination and running iterations,
        only the last X samples are stored (in other words samples are overwritten).
        The default history size set with `max_history` is used when calling
        `report_image`, `report_media` etc. without specifying `max_history`

        :param max_history: Number of samples (files) to store on a unique set of title/series being reported
            with different iteration counter. This is used to make sure users do not end up exploding storage
            on server storage side.

            For example the following code sample will store the last 5 images even though
            we are reporting 100 samples.

            .. code-block:: py

                logger.set_default_debug_sample_history(5)
                for i in range(100):
                    logger.report_image(title='image', series='sample', iteration=i, ...)

        :return:
        """
        self._default_max_sample_history = int(max_history)

    def get_default_debug_sample_history(self):
        # type: () -> int
        """
        Return the default max debug sample history when reporting media/debug samples.
        If value was not set specifically, the functions returns the configuration file default value.

        :return: default number of samples (files) to store on a unique set of title/series being reported
            with different iteration counter. This is used to make sure users do not end up exploding storage
            on server storage side.
        """
        if self._default_max_sample_history is not None:
            return self._default_max_sample_history
        # noinspection PyProtectedMember
        return int(UploadEvent._file_history_size)

    def report_image_and_upload(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            path=None,  # type: Optional[str]
            matrix=None,  # type: Optional[Union[np.ndarray, Image.Image]]
            max_image_history=None,  # type: Optional[int]
            delete_after_upload=False  # type: bool
    ):
        """
        .. deprecated:: 0.13.0
            Use :meth:`Logger.report_image` instead
        """
        self.report_image(title=title, series=series, iteration=iteration or 0, local_path=path, image=matrix,
                          max_image_history=max_image_history, delete_after_upload=delete_after_upload)

    def capture_logging(self):
        # type: () -> "_LoggingContext"
        """
        Return context capturing all the logs (via logging) reported under the context

        :return: a ContextManager
        """
        class _LoggingContext(object):
            def __init__(self, a_logger):
                self.logger = a_logger

            def __enter__(self, *_, **__):
                if not self.logger:
                    return
                StdStreamPatch.patch_logging_formatter(self.logger)

            def __exit__(self, *_, **__):
                if not self.logger:
                    return
                StdStreamPatch.remove_patch_logging_formatter()

        # Do nothing if we already have full logging support
        return _LoggingContext(None if self._connect_logging else self)

    @classmethod
    def tensorboard_auto_group_scalars(cls, group_scalars=False):
        # type: (bool) -> None
        """
        Group together TensorBoard scalars that do not have a title, or assign a title/series with the same tag.

        :param group_scalars: Group TensorBoard scalars without a title

            The values are:

            - ``True`` - Scalars without specific titles are grouped together in the "Scalars" plot, preserving
              backward compatibility with ClearML automagical behavior.
            - ``False`` - TensorBoard scalars without titles get a title/series with the same tag. (default)
        """
        cls._tensorboard_logging_auto_group_scalars = group_scalars

    @classmethod
    def tensorboard_single_series_per_graph(cls, single_series=False):
        # type: (bool) -> None
        """
        Deprecated, this is now controlled from the UI!
        Group TensorBoard scalar series together or in separate plots.

        :param single_series: Group TensorBoard scalar series together

            The values are:

            - ``True`` - Generate a separate plot for each TensorBoard scalar series.
            - ``False`` - Group the TensorBoard scalar series together in the same plot. (default)

        """
        cls._tensorboard_single_series_per_graph = single_series

    @classmethod
    def matplotlib_force_report_non_interactive(cls, force):
        # type: (bool) -> None
        """
        If True all matplotlib are always converted to non interactive static plots (images), appearing in under
        the Plots section. If False (default), matplotlib figures are converted into interactive web UI plotly
        figures, in case figure conversion fails, it defaults to non-interactive plots.

        :param force: If True all matplotlib figures are converted automatically to non-interactive plots.
        """
        from clearml.backend_interface.metrics import Reporter
        Reporter.matplotlib_force_report_non_interactive(force=force)

    @classmethod
    def set_reporting_nan_value(cls, value, warn_period=1000):
        # type: (float, int) -> None
        """
        When a NaN value is encountered, it is reported as a floating value (by default 0) and the user is warned.
        This function is used to change the value NaN is converted to and the warning period.

        :param value: The value NaN is converted to
        :param warn_period: Number of times NaN is encountered and converted until the next warning
        """
        MetricsEventAdapter.default_nan_value = value
        MetricsEventAdapter.report_nan_warning_period = warn_period

    @classmethod
    def set_reporting_inf_value(cls, value, warn_period=1000):
        # type: (float, int) -> None
        """
        When an inf value is encountered, it is reported as a floating value (by default 0) and the user is warned.
        This function is used to change the value inf is converted to and the warning period.

        :param value: The value inf is converted to
        :param warn_period: Number of times inf is encountered and converted until the next warning
        """
        MetricsEventAdapter.default_inf_value = value
        MetricsEventAdapter.report_inf_warning_period = warn_period

    @classmethod
    def _remove_std_logger(cls):
        # noinspection PyBroadException
        try:
            StdStreamPatch.remove_std_logger()
        except Exception:
            return False
        return True

    def _parse_level(self, level):
        # type: (Any) -> int
        try:
            return int(level)
        except (TypeError, ValueError):
            self._task.log.log(level=logging.ERROR,
                               msg='Logger failed casting log level "%s" to integer' % str(level))
            return logging.INFO

    def _console(self, msg, level=logging.INFO, omit_console=False, force_send=False, *args, **_):
        # type: (str, int, bool, bool, Any, Any) -> None
        """
        print text to log (same as print to console, and also prints to console)

        :param msg: text to print to the console (always send to the backend and displayed in console)
        :param level: logging level, default: logging.INFO
        :param omit_console: Omit the console output, and only send the ``msg`` value to the log
        :param force_send: Report with an explicit log level. Only supported if ``omit_console`` is True

            - ``True`` - Omit the console output.
            - ``False`` - Print the console output. (default)

        """
        level = self._parse_level(level)

        force_send = force_send and omit_console

        # noinspection PyProtectedMember
        if not self._skip_console_log() or not self._task._is_remote_main_task() or force_send:
            # check if we have a TaskHandler and that it is valid (shutdown will clear the .task_id)
            if self._task_handler and self._task_handler.task_id:
                # noinspection PyBroadException
                try:
                    record = self._task.log.makeRecord(
                        "console", level=level, fn='', lno=0, func='', msg=msg, args=args, exc_info=None
                    )
                    # find the task handler that matches our task
                    self._task_handler.emit(record)
                except Exception:
                    # avoid infinite loop, output directly to stderr
                    # noinspection PyBroadException
                    try:
                        # make sure we are writing to the original stdout
                        StdStreamPatch.stderr_original_write(
                            'clearml.Logger failed sending log [level {}]: "{}"\n'.format(level, msg))
                    except Exception:
                        pass
            else:
                # noinspection PyProtectedMember
                self._task._reporter.report_console(message=msg, level=level)

        if not omit_console:
            # if we are here and we grabbed the stdout, we need to print the real thing
            if self._connect_std_streams and not self._skip_console_log():
                # noinspection PyBroadException
                try:
                    # make sure we are writing to the original stdout
                    StdStreamPatch.stdout_original_write(str(msg) + '\n')
                except Exception:
                    pass
            else:
                print(str(msg))

        # if task was not started, we have to start it
        self._start_task_if_needed()

    def _report_image_plot_and_upload(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            path=None,  # type: Optional[str]
            matrix=None,  # type: Optional[np.ndarray]
            max_image_history=None,  # type: Optional[int]
            delete_after_upload=False  # type: bool
    ):
        """
        Report an image, upload its contents, and present in plots section using plotly

        Image is uploaded to a preconfigured bucket (see :meth:`Logger.set_default_upload_destination`)
        describing the task ID, title, series and iteration.

        :param title: Title (AKA metric)
        :param series: Series (AKA variant)
        :param iteration: Iteration number
        :param path: A path to an image file. Required unless matrix is provided.
        :param matrix: A 3D numpy.ndarray object containing image data (RGB). Required unless filename is provided.
        :param max_image_history: maximum number of image to store per metric/variant combination \
        use negative value for unlimited. default is set in global configuration (default=5)
        :param delete_after_upload: if True, one the file was uploaded the local copy will be deleted
        """

        # if task was not started, we have to start it
        self._start_task_if_needed()
        upload_uri = self.get_default_upload_destination()
        if not upload_uri:
            upload_uri = Path(get_cache_dir()) / 'debug_images'
            upload_uri.mkdir(parents=True, exist_ok=True)
            # Verify that we can upload to this destination
            upload_uri = str(upload_uri)
            storage = StorageHelper.get(upload_uri)
            upload_uri = storage.verify_upload(folder_uri=upload_uri)

        # noinspection PyProtectedMember
        self._task._reporter.report_image_plot_and_upload(
            title=title,
            series=series,
            path=path,
            matrix=matrix,
            iter=iteration or 0,
            upload_uri=upload_uri,
            max_image_history=max_image_history if max_image_history is not None else self._default_max_sample_history,
            delete_after_upload=delete_after_upload,
        )

    def _report_file_and_upload(
            self,
            title,  # type: str
            series,  # type: str
            iteration=None,  # type: Optional[int]
            path=None,  # type: Optional[str]
            max_file_history=None,  # type: Optional[int]
            delete_after_upload=False  # type: bool
    ):
        """
        Upload a file and report it as link in the debug images section.

        File is uploaded to a preconfigured storage (see :meth:`Loggerset_default_upload_destination`)
        describing the task ID, title, series and iteration.

        :param title: Title (AKA metric)
        :param series: Series (AKA variant)
        :param iteration: Iteration number
        :param path: A path to file to be uploaded
        :param max_file_history: maximum number of files to store per metric/variant combination \
        use negative value for unlimited. default is set in global configuration (default=5)
        :param delete_after_upload: if True, one the file was uploaded the local copy will be deleted
        """

        # if task was not started, we have to start it
        self._start_task_if_needed()
        upload_uri = self.get_default_upload_destination()
        if not upload_uri:
            upload_uri = Path(get_cache_dir()) / 'debug_images'
            upload_uri.mkdir(parents=True, exist_ok=True)
            # Verify that we can upload to this destination
            upload_uri = str(upload_uri)
            storage = StorageHelper.get(upload_uri)
            upload_uri = storage.verify_upload(folder_uri=upload_uri)
        # noinspection PyProtectedMember
        self._task._reporter.report_image_and_upload(
            title=title,
            series=series,
            path=path,
            image=None,
            iter=iteration or 0,
            upload_uri=upload_uri,
            max_image_history=max_file_history if max_file_history is not None else self._default_max_sample_history,
            delete_after_upload=delete_after_upload,
        )

    def _start_task_if_needed(self):
        # deprecated
        pass

    def _flush_stdout_handler(self):
        if self._task_handler:
            self._task_handler.flush()

    def _close_stdout_handler(self, wait=True):
        # detach the sys stdout/stderr
        if self._connect_std_streams:
            StdStreamPatch.remove_std_logger(self)

        if self._task_handler:
            t = self._task_handler
            self._task_handler = None
            t.close(wait)

    def _touch_title_series(self, title, series):
        # type: (str, str) -> None
        if title not in self._graph_titles:
            self._graph_titles[title] = set()
        self._graph_titles[title].add(series)

    def _get_used_title_series(self):
        # type: () -> dict
        return self._graph_titles

    def _get_tensorboard_series_prefix(self):
        # type: () -> Optional[str]
        """
        :return str: return a string prefix to put in front of every report combing from tensorboard
        """
        return self._tensorboard_series_force_prefix

    def _set_tensorboard_series_prefix(self, prefix):
        # type: (Optional[str]) -> ()
        """
        :param str prefix: Set a string prefix to put in front of every report combing from tensorboard
        """
        self._tensorboard_series_force_prefix = str(prefix) if prefix else None

    @classmethod
    def _get_tensorboard_auto_group_scalars(cls):
        # type: () -> bool
        """
        :return: True, if we preserve Tensorboard backward compatibility behaviour,
            i.e., scalars without specific title will be under the "Scalars" graph
            default is False: Tensorboard scalars without title will have title/series with the same tag
        """
        return cls._tensorboard_logging_auto_group_scalars

    @classmethod
    def _get_tensorboard_single_series_per_graph(cls):
        # type: () -> bool
        """
        :return: True, if we generate a separate graph (plot) for each Tensorboard scalar series
            default is False: Tensorboard scalar series will be grouped according to their title
        """
        return cls._tensorboard_single_series_per_graph

    @classmethod
    def _skip_console_log(cls):
        return bool(running_remotely() and not DEBUG_SIMULATE_REMOTE_TASK.get())


# clearml/model.py

import abc
import os
import tarfile
import zipfile
from tempfile import mkdtemp, mkstemp

import six
from typing import List, Dict, Union, Optional, Mapping, TYPE_CHECKING, Sequence, Any

from .backend_api import Session
from .backend_api.services import models, projects
from pathlib2 import Path

from .utilities.config import config_dict_to_text, text_to_config_dict
from .utilities.proxy_object import cast_basic_type

from .backend_interface.util import (
    validate_dict, get_single_result, mutually_exclusive, exact_match_regex,
    get_or_create_project, )
from .debugging.log import get_logger
from .storage.cache import CacheManager
from .storage.helper import StorageHelper
from .storage.util import get_common_path
from .utilities.enum import Options
from .backend_interface import Task as _Task
from .backend_interface.model import create_dummy_model, Model as _Model
from .backend_interface.session import SendError
from .config import running_remotely, get_cache_dir


if TYPE_CHECKING:
    from .task import Task


class Framework(Options):
    """
    Optional frameworks for output model
    """
    tensorflow = 'TensorFlow'
    tensorflowjs = 'TensorFlow_js'
    tensorflowlite = 'TensorFlow_Lite'
    pytorch = 'PyTorch'
    caffe = 'Caffe'
    caffe2 = 'Caffe2'
    onnx = 'ONNX'
    keras = 'Keras'
    mknet = 'MXNet'
    cntk = 'CNTK'
    torch = 'Torch'
    darknet = 'Darknet'
    paddlepaddle = 'PaddlePaddle'
    scikitlearn = 'ScikitLearn'
    xgboost = 'XGBoost'
    lightgbm = 'LightGBM'
    parquet = 'Parquet'
    megengine = 'MegEngine'
    catboost = 'CatBoost'

    __file_extensions_mapping = {
        '.pb': (tensorflow, tensorflowjs, onnx, ),
        '.meta': (tensorflow, ),
        '.pbtxt': (tensorflow, onnx, ),
        '.zip': (tensorflow, ),
        '.tgz': (tensorflow, ),
        '.tar.gz': (tensorflow, ),
        'model.json': (tensorflowjs, ),
        '.tflite': (tensorflowlite, ),
        '.pth': (pytorch, ),
        '.pt': (pytorch, ),
        '.caffemodel': (caffe, ),
        '.prototxt': (caffe, ),
        'predict_net.pb': (caffe2, ),
        'predict_net.pbtxt': (caffe2, ),
        '.onnx': (onnx, ),
        '.h5': (keras, ),
        '.hdf5': (keras, ),
        '.keras': (keras, ),
        '.model': (mknet, cntk, xgboost),
        '-symbol.json': (mknet, ),
        '.cntk': (cntk, ),
        '.t7': (torch, ),
        '.cfg': (darknet, ),
        '__model__': (paddlepaddle, ),
        '.pkl': (scikitlearn, keras, xgboost, megengine),
        '.parquet': (parquet, ),
        '.cbm': (catboost, ),
    }

    __parent_mapping = {
        "tensorflow": (
            tensorflow,
            tensorflowjs,
            tensorflowlite,
            keras,
        ),
        "pytorch": (pytorch,),
        "xgboost": (xgboost,),
        "lightgbm": (lightgbm,),
        "catboost": (catboost,),
        "joblib": (scikitlearn, xgboost)
    }

    @classmethod
    def get_framework_parents(cls, framework):
        if not framework:
            return []
        parents = []
        for k, v in cls.__parent_mapping.items():
            if framework in v:
                parents.append(k)
        return parents

    @classmethod
    def _get_file_ext(cls, framework, filename):
        mapping = cls.__file_extensions_mapping
        filename = filename.lower()

        def find_framework_by_ext(framework_selector):
            for ext, frameworks in mapping.items():
                if frameworks and filename.endswith(ext):
                    fw = framework_selector(frameworks)
                    if fw:
                        return fw, ext

        # If no framework, try finding first framework matching the extension, otherwise (or if no match) try matching
        # the given extension to the given framework. If no match return an empty extension
        return (
            (not framework and find_framework_by_ext(lambda frameworks_: frameworks_[0]))
            or find_framework_by_ext(lambda frameworks_: framework if framework in frameworks_ else None)
            or (framework, filename.split('.')[-1] if '.' in filename else '')
        )


@six.add_metaclass(abc.ABCMeta)
class BaseModel(object):
    # noinspection PyProtectedMember
    _archived_tag = _Task.archived_tag
    _package_tag = "package"

    @property
    def id(self):
        # type: () -> str
        """
        The Id (system UUID) of the model.

        :return: The model ID.
        """
        return self._get_model_data().id

    @property
    def name(self):
        # type: () -> str
        """
        The name of the model.

        :return: The model name.
        """
        return self._get_model_data().name

    @name.setter
    def name(self, value):
        # type: (str) -> None
        """
        Set the model name.

        :param str value: The model name.
        """
        self._get_base_model().update(name=value)

    @property
    def project(self):
        # type: () -> str
        """
        project id of the model.

        :return: project id (str).
        """
        data = self._get_model_data()
        return data.project

    @project.setter
    def project(self, value):
        # type: (str) -> None
        """
        Set the project ID of the model.

        :param value: project ID (str).

        :type value: str
        """
        self._get_base_model().update(project_id=value)

    @property
    def comment(self):
        # type: () -> str
        """
        The comment for the model. Also, use for a model description.

        :return: The model comment / description.
        """
        return self._get_model_data().comment

    @comment.setter
    def comment(self, value):
        # type: (str) -> None
        """
        Set comment for the model. Also, use for a model description.

        :param str value: The model comment/description.
        """
        self._get_base_model().update(comment=value)

    @property
    def tags(self):
        # type: () -> List[str]
        """
        A list of tags describing the model.

        :return: The list of tags.
        """
        return self._get_model_data().tags

    @tags.setter
    def tags(self, value):
        # type: (List[str]) -> None
        """
        Set the list of tags describing the model.

        :param value: The tags.

        :type value: list(str)
        """
        self._get_base_model().update(tags=value)

    @property
    def system_tags(self):
        # type: () -> List[str]
        """
        A list of system tags describing the model.

        :return: The list of tags.
        """
        data = self._get_model_data()
        return data.system_tags if Session.check_min_api_version('2.3') else data.tags

    @system_tags.setter
    def system_tags(self, value):
        # type: (List[str]) -> None
        """
        Set the list of system tags describing the model.

        :param value: The tags.

        :type value: list(str)
        """
        self._get_base_model().update(system_tags=value)

    @property
    def config_text(self):
        # type: () -> str
        """
        The configuration as a string. For example, prototxt, an ini file, or Python code to evaluate.

        :return: The configuration.
        """
        # noinspection PyProtectedMember
        return _Model._unwrap_design(self._get_model_data().design)

    @property
    def config_dict(self):
        # type: () -> dict
        """
        The configuration as a dictionary, parsed from the design text. This usually represents the model configuration.
        For example, prototxt, an ini file, or Python code to evaluate.

        :return: The configuration.
        """
        return self._text_to_config_dict(self.config_text)

    @property
    def labels(self):
        # type: () -> Dict[str, int]
        """
        The label enumeration of string (label) to integer (value) pairs.


        :return: A dictionary containing labels enumeration, where the keys are labels and the values as integers.
        """
        return self._get_model_data().labels

    @property
    def task(self):
        # type: () -> str
        """
        Return the creating task ID

        :return: The Task ID (str)
        """
        return self._task.id if self._task else self._get_base_model().task

    @property
    def url(self):
        # type: () -> str
        """
        Return the url of the model file (or archived files)

        :return: The model file URL.
        """
        return self._get_base_model().uri

    @property
    def published(self):
        # type: () -> bool
        return self._get_base_model().locked

    @property
    def framework(self):
        # type: () -> str
        return self._get_model_data().framework

    def __init__(self, task=None):
        # type: (Task) -> None
        super(BaseModel, self).__init__()
        self._log = get_logger()
        self._task = None
        self._set_task(task)

    def get_weights(self, raise_on_error=False):
        # type: (bool) -> str
        """
        Download the base model and return the locally stored filename.

        :param bool raise_on_error: If True and the artifact could not be downloaded,
            raise ValueError, otherwise return None on failure and output log warning.

        :return: The locally stored file.
        """
        # download model (synchronously) and return local file
        return self._get_base_model().download_model_weights(raise_on_error=raise_on_error)

    def get_weights_package(self, return_path=False, raise_on_error=False):
        # type: (bool, bool) -> Optional[Union[str, List[Path]]]
        """
        Download the base model package into a temporary directory (extract the files), or return a list of the
        locally stored filenames.

        :param bool return_path: Return the model weights or a list of filenames (Optional)

            - ``True`` - Download the model weights into a temporary directory, and return the temporary directory path.
            - ``False`` - Return a list of the locally stored filenames. (Default)

        :param bool raise_on_error: If True and the artifact could not be downloaded,
            raise ValueError, otherwise return None on failure and output log warning.

        :return: The model weights, or a list of the locally stored filenames.
            if raise_on_error=False, returns None on error.
        """
        # check if model was packaged
        if not self._is_package():
            raise ValueError('Model is not packaged')

        # download packaged model
        packed_file = self.get_weights(raise_on_error=raise_on_error)

        if not packed_file:
            if raise_on_error:
                raise ValueError('Model package \'{}\' could not be downloaded'.format(self.url))
            return None

        # unpack
        target_folder = mkdtemp(prefix='model_package_')
        if not target_folder:
            raise ValueError('cannot create temporary directory for packed weight files')

        for func in (zipfile.ZipFile, tarfile.open):
            try:
                obj = func(packed_file)
                obj.extractall(path=target_folder)
                break
            except (zipfile.BadZipfile, tarfile.ReadError):
                pass
        else:
            raise ValueError('cannot extract files from packaged model at %s', packed_file)

        if return_path:
            return target_folder

        target_files = list(Path(target_folder).glob('*'))
        return target_files

    def publish(self):
        # type: () -> ()
        """
        Set the model to the status ``published`` and for public use. If the model's status is already ``published``,
        then this method is a no-op.
        """

        if not self.published:
            self._get_base_model().publish()

    def _running_remotely(self):
        # type: () -> ()
        return bool(running_remotely() and self._task is not None)

    def _set_task(self, value):
        # type: (_Task) -> ()
        if value is not None and not isinstance(value, _Task):
            raise ValueError('task argument must be of Task type')
        self._task = value

    @abc.abstractmethod
    def _get_model_data(self):
        pass

    @abc.abstractmethod
    def _get_base_model(self):
        pass

    def _set_package_tag(self):
        if self._package_tag not in self.system_tags:
            self.system_tags.append(self._package_tag)
            self._get_base_model().edit(system_tags=self.system_tags)

    def _is_package(self):
        return self._package_tag in (self.system_tags or [])

    @staticmethod
    def _config_dict_to_text(config):
        if not isinstance(config, six.string_types) and not isinstance(config, dict):
            raise ValueError("Model configuration only supports dictionary or string objects")
        return config_dict_to_text(config)

    @staticmethod
    def _text_to_config_dict(text):
        if not isinstance(text, six.string_types):
            raise ValueError("Model configuration parsing only supports string")
        return text_to_config_dict(text)

    @staticmethod
    def _resolve_config(config_text=None, config_dict=None):
        mutually_exclusive(config_text=config_text, config_dict=config_dict, _require_at_least_one=False)
        if config_dict:
            return InputModel._config_dict_to_text(config_dict)

        return config_text


class Model(BaseModel):
    """
    Represent an existing model in the system, search by model id.
    The Model will be read-only and can be used to pre initialize a network
    """

    def __init__(self, model_id):
        # type: (str) ->None
        """
        Load model based on id, returned object is read-only and can be connected to a task

        Notice, we can override the input model when running remotely

        :param model_id: id (string)
        """
        super(Model, self).__init__()
        self._base_model_id = model_id
        self._base_model = None
        self._reload_required = False

    def get_local_copy(self, extract_archive=True, raise_on_error=False):
        # type: (bool, bool) -> str
        """
        Retrieve a valid link to the model file(s).
        If the model URL is a file system link, it will be returned directly.
        If the model URL is points to a remote location (http/s3/gs etc.),
        it will download the file(s) and return the temporary location of the downloaded model.

        :param bool extract_archive: If True and the model is of type 'packaged' (e.g. TensorFlow compressed folder)
            The returned path will be a temporary folder containing the archive content
        :param bool raise_on_error: If True and the artifact could not be downloaded,
            raise ValueError, otherwise return None on failure and output log warning.

        :return: A local path to the model (or a downloaded copy of it).
        """
        if extract_archive and self._is_package():
            return self.get_weights_package(return_path=True, raise_on_error=raise_on_error)
        return self.get_weights(raise_on_error=raise_on_error)

    def _get_base_model(self):
        if self._base_model:
            return self._base_model

        if not self._base_model_id:
            # this shouldn't actually happen
            raise Exception('Missing model ID, cannot create an empty model')
        self._base_model = _Model(
            upload_storage_uri=None,
            cache_dir=get_cache_dir(),
            model_id=self._base_model_id,
        )
        return self._base_model

    def _get_model_data(self):
        return self._get_base_model().data

    @classmethod
    def query_models(
            cls,
            project_name=None,  # type: Optional[str]
            model_name=None,  # type: Optional[str]
            tags=None,  # type: Optional[Sequence[str]]
            only_published=False,  # type: bool
            include_archived=False,  # type: bool
            max_results=None,  # type: Optional[int]
    ):
        # type: (...) -> List[Model]
        """
        Return Model objects from the project artifactory.
        Filter based on project-name / model-name / tags.
        List is always returned sorted by descending last update time (i.e. latest model is the first in the list)

        :param project_name: Optional, filter based project name string, if not given query models from all projects
        :param model_name: Optional Model name as shown in the model artifactory
        :param tags: Optional filter models based on list of tags, example: ['production', 'verified', '-qa']
            Notice use '-' prefix to filter out tags.
        :param only_published: If True only return published models.
        :param include_archived: If True return archived models.
        :param max_results: Optional return the last X models,
            sorted by last update time (from the most recent to the least).

        :return: ModeList of Models objects
        """
        if project_name:
            # noinspection PyProtectedMember
            res = _Model._get_default_session().send(
                projects.GetAllRequest(
                    name=exact_match_regex(project_name),
                    only_fields=['id', 'name', 'last_update']
                )
            )
            project = get_single_result(entity='project', query=project_name, results=res.response.projects)
        else:
            project = None

        only_fields = ['id', 'created', 'system_tags']

        # noinspection PyProtectedMember
        res = _Model._get_default_session().send(
            models.GetAllRequest(
                project=[project.id] if project else None,
                name=exact_match_regex(model_name) if model_name is not None else None,
                only_fields=only_fields,
                tags=tags or None,
                system_tags=["-" + cls._archived_tag] if not include_archived else None,
                ready=True if only_published else None,
                order_by=['-created'],
                page=0 if max_results else None,
                page_size=max_results or None,
            )
        )
        if not res.response.models:
            return []

        return [Model(model_id=m.id) for m in res.response.models]

    @property
    def id(self):
        # type: () -> str
        return self._base_model_id if self._base_model_id else super(Model, self).id

    @classmethod
    def remove(cls, model, delete_weights_file=True, force=False, raise_on_errors=False):
        # type: (Union[str, Model], bool, bool, bool) -> bool
        """
        Remove a model from the model repository.
        Optional, delete the model weights file from the remote storage.

        :param model: Model ID or Model object to remove
        :param delete_weights_file: If True (default) delete the weights file from the remote storage
        :param force: If True, remove model even if other Tasks are using this model. default False.
        :param raise_on_errors: If True, throw ValueError if something went wrong, default False.
        :return: True if Model was removed successfully
            partial removal returns False, i.e. Model was deleted but weights file deletion failed
        """
        if isinstance(model, str):
            model = Model(model_id=model)

        # noinspection PyBroadException
        try:
            weights_url = model.url
        except Exception:
            if raise_on_errors:
                raise ValueError("Could not find model id={}".format(model.id))
            return False

        try:
            # noinspection PyProtectedMember
            res = _Model._get_default_session().send(
                models.DeleteRequest(model.id, force=force),
            )
            response = res.wait()
            if not response.ok():
                if raise_on_errors:
                    raise ValueError("Could not remove model id={}: {}".format(model.id, response.meta))
                return False
        except SendError as ex:
            if raise_on_errors:
                raise ValueError("Could not remove model id={}: {}".format(model.id, ex))
            return False
        except ValueError:
            if raise_on_errors:
                raise
            return False
        except Exception as ex:
            if raise_on_errors:
                raise ValueError("Could not remove model id={}: {}".format(model.id, ex))
            return False

        if not delete_weights_file:
            return True

        helper = StorageHelper.get(url=weights_url)
        try:
            if not helper.delete(weights_url):
                if raise_on_errors:
                    raise ValueError("Could not remove model id={} weights file: {}".format(model.id, weights_url))
                return False
        except Exception as ex:
            if raise_on_errors:
                raise ValueError("Could not remove model id={} weights file \'{}\': {}".format(
                    model.id, weights_url, ex))
            return False

        return True

    def set_metadata(self, key, value, type):
        # type: (str, str, str) -> bool
        """
        Set one metadata entry. All parameters must be strings or castable to strings

        :param key: Key of the metadata entry
        :param value: Value of the metadata entry
        :param type: Type of the metadata entry

        :return: True if the metadata was set and False otherwise
        """
        self._reload_required = True
        result = _Model._get_default_session().send(models.AddOrUpdateMetadataRequest(
            metadata=[{"key": str(key), "value": str(value), "type": str(type)}],
            model=self._base_model_id,
            replace_metadata=False
        ))
        return bool(result)

    def get_metadata(self, key):
        # type: (str) -> Optional[str]
        """
        Get one metadata entry value (as a string) based on its key. See `Model.get_metadata_casted`
        if you wish to cast the value to its type (if possible)

        :param key: Key of the metadata entry you want to get

        :return: String representation of the value of the metadata entry or None if the entry was not found
        """
        self._reload_if_required()
        return self.get_all_metadata().get(str(key), {}).get("value")

    def get_metadata_casted(self, key):
        # type: (str) -> Optional[str]
        """
        Get one metadata entry based on its key, casted to its type if possible

        :param key: Key of the metadata entry you want to get

        :return: The value of the metadata entry, casted to its type (if not possible,
            the string representation will be returned) or None if the entry was not found
        """
        key = str(key)
        metadata = self.get_all_metadata()
        if key not in metadata:
            return None
        return cast_basic_type(metadata[key].get("value"), metadata[key].get("type"))

    def get_all_metadata(self):
        # type: () -> Dict[str, Dict[str, str]]
        """
        See `Model.get_all_metadata_casted` if you wish to cast the value to its type (if possible)

        :return: Get all metadata as a dictionary of format Dict[key, Dict[value, type]]. The key, value and type
            entries are all strings. Note that each entry might have an additional 'key' entry, repeating the key
        """
        self._reload_if_required()
        return self._get_model_data().metadata or {}

    def get_all_metadata_casted(self):
        # type: () -> Dict[str, Dict[str, Any]]
        """
        :return: Get all metadata as a dictionary of format Dict[key, Dict[value, type]]. The key and type
            entries are strings. The value is cast to its type if possible. Note that each entry might
            have an additional 'key' entry, repeating the key
        """
        self._reload_if_required()
        result = {}
        metadata = self.get_all_metadata()
        for key, metadata_entry in metadata.items():
            result[key] = cast_basic_type(metadata_entry.get("value"), metadata_entry.get("type"))
        return result

    def set_all_metadata(self, metadata, replace=True):
        # type: (Dict[str, Dict[str, str]], bool) -> bool
        """
        Set metadata based on the given parameters. Allows replacing all entries or updating the current entries.

        :param metadata: A dictionary of format Dict[key, Dict[value, type]] representing the metadata you want to set
        :param replace: If True, replace all metadata with the entries in the `metadata` parameter. If False,
            keep the old metadata and update it with the entries in the `metadata` parameter (add or change it)

        :return: True if the metadata was set and False otherwise
        """
        self._reload_required = True
        metadata_array = [
            {"key": str(k), "value": str(v_t.get("value")), "type": str(v_t.get("type"))} for k, v_t in metadata.items()
        ]
        result = _Model._get_default_session().send(models.AddOrUpdateMetadataRequest(
            metadata=metadata_array,
            model=self._base_model_id,
            replace_metadata=replace
        ))
        return bool(result)

    def _reload_if_required(self):
        if not self._base_model:
            self._get_base_model()
        if self._reload_required:
            self._base_model.reload()
            self._reload_required = False


class InputModel(Model):
    """
    Load an existing model in the system, search by model id.
    The Model will be read-only and can be used to pre initialize a network
    We can connect the model to a task as input model, then when running remotely override it with the UI.
    """

    # noinspection PyProtectedMember
    _EMPTY_MODEL_ID = _Model._EMPTY_MODEL_ID

    @classmethod
    def import_model(
        cls,
        weights_url,  # type: str
        config_text=None,  # type: Optional[str]
        config_dict=None,  # type: Optional[dict]
        label_enumeration=None,  # type: Optional[Mapping[str, int]]
        name=None,  # type: Optional[str]
        project=None,  # type: Optional[str]
        tags=None,  # type: Optional[List[str]]
        comment=None,  # type: Optional[str]
        is_package=False,  # type: bool
        create_as_published=False,  # type: bool
        framework=None,  # type: Optional[str]
    ):
        # type: (...) -> InputModel
        """
        Create an InputModel object from a pre-trained model by specifying the URL of an initial weight files.
        Optionally, input a configuration, label enumeration, name for the model, tags describing the model,
        comment as a description of the model, indicate whether the model is a package, specify the model's
        framework, and indicate whether to immediately set the model's status to ``Published``.
        The model is read-only.

        The **ClearML Server** (backend) may already store the model's URL. If the input model's URL is not
        stored, meaning the model is new, then it is imported and ClearML stores its metadata.
        If the URL is already stored, the import process stops, ClearML issues a warning message, and ClearML
        reuses the model.

        In your Python experiment script, after importing the model, you can connect it to the main execution
        Task as an input model using :meth:`InputModel.connect` or :meth:`.Task.connect`. That initializes the
        network.

        .. note::
           Using the **ClearML Web-App** (user interface), you can reuse imported models and switch models in
           experiments.

        :param str weights_url: A valid URL for the initial weights file. If the **ClearML Web-App** (backend)
            already stores the metadata of a model with the same URL, that existing model is returned
            and ClearML ignores all other parameters.

            For example:

            - ``https://domain.com/file.bin``
            - ``s3://bucket/file.bin``
            - ``file:///home/user/file.bin``

        :param str config_text: The configuration as a string. This is usually the content of a configuration
            dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
        :type config_text: unconstrained text string
        :param dict config_dict: The configuration as a dictionary. Specify ``config_text`` or ``config_dict``,
            but not both.
        :param dict label_enumeration: Optional label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }
        :param str name: The name of the newly imported model. (Optional)
        :param str project: The project name to add the model into. (Optional)
        :param tags: The list of tags which describe the model. (Optional)
        :type tags: list(str)
        :param str comment: A comment / description for the model. (Optional)
        :type comment: str
        :param is_package: Is the imported weights file is a package (Optional)

            - ``True`` - Is a package. Add a package tag to the model.
            - ``False`` - Is not a package. Do not add a package tag. (Default)

        :type is_package: bool
        :param bool create_as_published: Set the model's status to Published (Optional)

            - ``True`` - Set the status to Published.
            - ``False`` - Do not set the status to Published. The status will be Draft. (Default)

        :param str framework: The framework of the model. (Optional)
        :type framework: str or Framework object

        :return: The imported model or existing model (see above).
        """
        config_text = cls._resolve_config(config_text=config_text, config_dict=config_dict)
        weights_url = StorageHelper.conform_url(weights_url)
        if not weights_url:
            raise ValueError("Please provide a valid weights_url parameter")
        # convert local to file to remote one
        weights_url = CacheManager.get_remote_url(weights_url)

        extra = {'system_tags': ["-" + cls._archived_tag]} \
            if Session.check_min_api_version('2.3') else {'tags': ["-" + cls._archived_tag]}
        # noinspection PyProtectedMember
        result = _Model._get_default_session().send(models.GetAllRequest(
            uri=[weights_url],
            only_fields=["id", "name", "created"],
            **extra
        ))

        if result.response.models:
            logger = get_logger()

            logger.debug('A model with uri "{}" already exists. Selecting it'.format(weights_url))

            model = get_single_result(
                entity='model',
                query=weights_url,
                results=result.response.models,
                log=logger,
                raise_on_error=False,
            )

            logger.info("Selected model id: {}".format(model.id))

            return InputModel(model_id=model.id)

        base_model = _Model(
            upload_storage_uri=None,
            cache_dir=get_cache_dir(),
        )

        from .task import Task
        task = Task.current_task()
        if task:
            comment = 'Imported by task id: {}'.format(task.id) + ('\n' + comment if comment else '')
            project_id = task.project
            name = name or 'Imported by {}'.format(task.name or '')
            # do not register the Task, because we do not want it listed after as "output model",
            # the Task never actually created the Model
            task_id = None
        else:
            project_id = None
            task_id = None

        if project:
            project_id = get_or_create_project(
                session=task.session if task else Task._get_default_session(),
                project_name=project
            )

        if not framework:
            # noinspection PyProtectedMember
            framework, file_ext = Framework._get_file_ext(
                framework=framework,
                filename=weights_url
            )

        base_model.update(
            design=config_text,
            labels=label_enumeration,
            name=name,
            comment=comment,
            tags=tags,
            uri=weights_url,
            framework=framework,
            project_id=project_id,
            task_id=task_id,
        )

        this_model = InputModel(model_id=base_model.id)
        this_model._base_model = base_model

        if is_package:
            this_model._set_package_tag()

        if create_as_published:
            this_model.publish()

        return this_model

    @classmethod
    def load_model(cls, weights_url, load_archived=False):
        # type: (str, bool) -> InputModel
        """
        Load an already registered model based on a pre-existing model file (link must be valid). If the url to the
        weights file already exists, the returned object is a Model representing the loaded Model. If no registered
        model with the specified url is found, ``None`` is returned.

        :param weights_url: The valid url for the weights file (string).

            Examples:

            .. code-block:: py

                "https://domain.com/file.bin" or "s3://bucket/file.bin" or "file:///home/user/file.bin".

            .. note::
                If a model with the exact same URL exists, it will be used, and all other arguments will be ignored.

        :param bool load_archived: Load archived models

            - ``True`` - Load the registered Model, if it is archived.
            - ``False`` - Ignore archive models.

        :return: The InputModel object, or None if no model could be found.
        """
        weights_url = StorageHelper.conform_url(weights_url)
        if not weights_url:
            raise ValueError("Please provide a valid weights_url parameter")

        # convert local to file to remote one
        weights_url = CacheManager.get_remote_url(weights_url)

        if not load_archived:
            # noinspection PyTypeChecker
            extra = {'system_tags': ["-" + _Task.archived_tag]} \
                if Session.check_min_api_version('2.3') else {'tags': ["-" + cls._archived_tag]}
        else:
            extra = {}

        # noinspection PyProtectedMember
        result = _Model._get_default_session().send(models.GetAllRequest(
            uri=[weights_url],
            only_fields=["id", "name", "created"],
            **extra
        ))

        if not result or not result.response or not result.response.models:
            return None

        logger = get_logger()
        model = get_single_result(
            entity='model',
            query=weights_url,
            results=result.response.models,
            log=logger,
            raise_on_error=False,
        )

        return InputModel(model_id=model.id)

    @classmethod
    def empty(cls, config_text=None, config_dict=None, label_enumeration=None):
        # type: (Optional[str], Optional[dict], Optional[Mapping[str, int]]) -> InputModel
        """
        Create an empty model object. Later, you can assign a model to the empty model object.

        :param config_text: The model configuration as a string. This is usually the content of a configuration
            dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
        :type config_text: unconstrained text string
        :param dict config_dict: The model configuration as a dictionary. Specify ``config_text`` or ``config_dict``,
            but not both.
        :param dict label_enumeration: The label enumeration dictionary of string (label) to integer (value) pairs.
            (Optional)

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        :return: An empty model object.
        """
        design = cls._resolve_config(config_text=config_text, config_dict=config_dict)

        this_model = InputModel(model_id=cls._EMPTY_MODEL_ID)
        this_model._base_model = m = _Model(
            cache_dir=None,
            upload_storage_uri=None,
            model_id=cls._EMPTY_MODEL_ID,
        )
        # noinspection PyProtectedMember
        m._data.design = _Model._wrap_design(design)
        # noinspection PyProtectedMember
        m._data.labels = label_enumeration
        return this_model

    def __init__(self, model_id=None, name=None, project=None, tags=None, only_published=False):
        # type: (Optional[str], Optional[str], Optional[str], Optional[Sequence[str]], bool) -> None
        """
        Load a model from the Model artifactory,
        based on model_id (uuid) or a model name/projects/tags combination.

        :param model_id: The ClearML Id (system UUID) of the input model whose metadata the **ClearML Server**
            (backend) stores. If provided all other arguments are ignored
        :param name: Model name to search and load
        :param project: Model project name to search model in
        :param tags: Model tags list to filter by
        :param only_published: If True filter out non-published (draft) models
        """
        if not model_id:
            found_models = self.query_models(
                project_name=project, model_name=name, tags=tags, only_published=only_published)
            if not found_models:
                raise ValueError("Could not locate model with project={} name={} tags={} published={}".format(
                    project, name, tags, only_published))
            model_id = found_models[0].id
        super(InputModel, self).__init__(model_id)

    @property
    def id(self):
        # type: () -> str
        return self._base_model_id

    def connect(self, task, name=None):
        # type: (Task, Optional[str]) -> None
        """
        Connect the current model to a Task object, if the model is preexisting. Preexisting models include:

        - Imported models (InputModel objects created using the :meth:`Logger.import_model` method).
        - Models whose metadata is already in the ClearML platform, meaning the InputModel object is instantiated
          from the ``InputModel`` class specifying the model's ClearML Id as an argument.
        - Models whose origin is not ClearML that are used to create an InputModel object. For example,
          models created using TensorFlow models.

        When the experiment is executed remotely in a worker, the input model already specified in the experiment is
        used.

        .. note::
           The **ClearML Web-App** allows you to switch one input model for another and then enqueue the experiment
           to execute in a worker.

        :param object task: A Task object.
        :param str name: The model name to be stored on the Task
            (default the filename, of the model weights, without the file extension)
        """
        self._set_task(task)

        model_id = None
        # noinspection PyProtectedMember
        if running_remotely() and (task.is_main_task() or task._is_remote_main_task()):
            input_models = task.input_models_id
            # noinspection PyBroadException
            try:
                # TODO: (temp fix) At the moment, the UI changes the key of the model hparam
                # when modifying its value... There is no way to tell which model was changed
                # so just take the first one in case `name` is not in `input_models`
                model_id = input_models.get(name, next(iter(input_models.values())))
                self._base_model_id = model_id
                self._base_model = InputModel(model_id=model_id)._get_base_model()
            except Exception:
                model_id = None

        if not model_id:
            # we should set the task input model to point to us
            model = self._get_base_model()
            # try to store the input model id, if it is not empty
            # (Empty Should not happen)
            if model.id != self._EMPTY_MODEL_ID:
                task.set_input_model(model_id=model.id, name=name)
            # only copy the model design if the task has no design to begin with
            # noinspection PyProtectedMember
            if not self._task._get_model_config_text() and model.model_design:
                # noinspection PyProtectedMember
                task._set_model_config(config_text=model.model_design)
            if not self._task.get_labels_enumeration() and model.data.labels:
                task.set_model_label_enumeration(model.data.labels)


class OutputModel(BaseModel):
    """
    Create an output model for a Task (experiment) to store the training results.

    The OutputModel object is always connected to a Task object, because it is instantiated with a Task object
    as an argument. It is, therefore, automatically registered as the Task's (experiment's) output model.

    The OutputModel object is read-write.

    A common use case is to reuse the OutputModel object, and override the weights after storing a model snapshot.
    Another use case is to create multiple OutputModel objects for a Task (experiment), and after a new high score
    is found, store a model snapshot.

    If the model configuration and / or the model's label enumeration
    are ``None``, then the output model is initialized with the values from the Task object's input model.

    .. note::
       When executing a Task (experiment) remotely in a worker, you can modify the model configuration and / or model's
       label enumeration using the **ClearML Web-App**.
    """

    _default_output_uri = None

    @property
    def published(self):
        # type: () -> bool
        """
        Get the published state of this model.

        :return:

        """
        if not self.id:
            return False
        return self._get_base_model().locked

    @property
    def config_text(self):
        # type: () -> str
        """
        Get the configuration as a string. For example, prototxt, an ini file, or Python code to evaluate.

        :return: The configuration.
        """
        # noinspection PyProtectedMember
        return _Model._unwrap_design(self._get_model_data().design)

    @config_text.setter
    def config_text(self, value):
        # type: (str) -> None
        """
        Set the configuration. Store a blob of text for custom usage.
        """
        self.update_design(config_text=value)

    @property
    def config_dict(self):
        # type: () -> dict
        """
        Get the configuration as a dictionary parsed from the ``config_text`` text. This usually represents the model
        configuration. For example, from prototxt to ini file or python code to evaluate.

        :return: The configuration.
        """
        return self._text_to_config_dict(self.config_text)

    @config_dict.setter
    def config_dict(self, value):
        # type: (dict) -> None
        """
        Set the configuration. Saved in the model object.

        :param dict value: The configuration parameters.
        """
        self.update_design(config_dict=value)

    @property
    def labels(self):
        # type: () -> Dict[str, int]
        """
        Get the label enumeration as a dictionary of string (label) to integer (value) pairs.

        For example:

        .. code-block:: javascript

           {
                'background': 0,
                'person': 1
           }

        :return: The label enumeration.
        """
        return self._get_model_data().labels

    @labels.setter
    def labels(self, value):
        # type: (Mapping[str, int]) -> None
        """
        Set the label enumeration.

        :param dict value: The label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        """
        self.update_labels(labels=value)

    @property
    def upload_storage_uri(self):
        # type: () -> str
        return self._get_base_model().upload_storage_uri

    def __init__(
        self,
        task=None,  # type: Optional[Task]
        config_text=None,  # type: Optional[str]
        config_dict=None,  # type: Optional[dict]
        label_enumeration=None,  # type: Optional[Mapping[str, int]]
        name=None,  # type: Optional[str]
        tags=None,  # type: Optional[List[str]]
        comment=None,  # type: Optional[str]
        framework=None,  # type: Optional[Union[str, Framework]]
        base_model_id=None,  # type: Optional[str]
    ):
        """
        Create a new model and immediately connect it to a task.

        We do not allow for Model creation without a task, so we always keep track on how we created the models
        In remote execution, Model parameters can be overridden by the Task
        (such as model configuration & label enumerator)

        :param task: The Task object with which the OutputModel object is associated.
        :type task: Task
        :param config_text: The configuration as a string. This is usually the content of a configuration
            dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
        :type config_text: unconstrained text string
        :param dict config_dict: The configuration as a dictionary.
            Specify ``config_dict`` or ``config_text``, but not both.
        :param dict label_enumeration: The label enumeration dictionary of string (label) to integer (value) pairs.
            (Optional)

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        :param str name: The name for the newly created model. (Optional)
        :param list(str) tags: A list of strings which are tags for the model. (Optional)
        :param str comment: A comment / description for the model. (Optional)
        :param framework: The framework of the model or a Framework object. (Optional)
        :type framework: str or Framework object
        :param base_model_id: optional, model id to be reused
        """
        if not task:
            from .task import Task
            task = Task.current_task()
            if not task:
                raise ValueError("task object was not provided, and no current task was found")

        super(OutputModel, self).__init__(task=task)

        config_text = self._resolve_config(config_text=config_text, config_dict=config_dict)

        self._model_local_filename = None
        self._last_uploaded_url = None
        self._base_model = None
        self._base_model_id = None
        self._task_connect_name = None
        # noinspection PyProtectedMember
        self._floating_data = create_dummy_model(
            design=_Model._wrap_design(config_text),
            labels=label_enumeration or task.get_labels_enumeration(),
            name=name or self._task.name,
            tags=tags,
            comment='{} by task id: {}'.format('Created' if not base_model_id else 'Overwritten', task.id) +
                    ('\n' + comment if comment else ''),
            framework=framework,
            upload_storage_uri=task.output_uri,
        )
        # If we have no real model ID, we are done
        if not base_model_id:
            return

        # noinspection PyBroadException
        try:
            # noinspection PyProtectedMember
            _base_model = self._task._get_output_model(model_id=base_model_id)
            _base_model.update(
                labels=self._floating_data.labels,
                design=self._floating_data.design,
                task_id=self._task.id,
                project_id=self._task.project,
                name=self._floating_data.name or self._task.name,
                comment=('{}\n{}'.format(_base_model.comment, self._floating_data.comment)
                         if (_base_model.comment and self._floating_data.comment and
                             self._floating_data.comment not in _base_model.comment)
                         else (_base_model.comment or self._floating_data.comment)),
                tags=self._floating_data.tags,
                framework=self._floating_data.framework,
                upload_storage_uri=self._floating_data.upload_storage_uri
            )
            self._base_model = _base_model
            self._floating_data = None
            name = self._task_connect_name or Path(_base_model.uri).stem
        except Exception:
            pass
        self.connect(task, name=name)

    def connect(self, task, name=None):
        # type: (Task, Optional[str]) -> None
        """
        Connect the current model to a Task object, if the model is a preexisting model. Preexisting models include:

        - Imported models.
        - Models whose metadata the **ClearML Server** (backend) is already storing.
        - Models from another source, such as frameworks like TensorFlow.

        :param object task: A Task object.
        :param str name: The model name as it would appear on the Task object.
            The model object itself can have a different name,
            this is designed to support multiple models used/created by a single Task.
            Use examples would be GANs or model ensemble
        """
        if self._task != task:
            raise ValueError('Can only connect preexisting model to task, but this is a fresh model')

        if name:
            self._task_connect_name = name

        # we should set the task input model to point to us
        model = self._get_base_model()

        # only copy the model design if the task has no design to begin with
        # noinspection PyProtectedMember
        if not self._task._get_model_config_text():
            # noinspection PyProtectedMember
            task._set_model_config(
                config_text=model.model_design if hasattr(model, "model_design") else model.design.get("design", "")
            )
        if not self._task.get_labels_enumeration():
            task.set_model_label_enumeration(model.data.labels if hasattr(model, "data") else model.labels)

        if self._base_model:
            self._base_model.update_for_task(
                task_id=self._task.id, model_id=self.id, type_="output", name=self._task_connect_name)

    def set_upload_destination(self, uri):
        # type: (str) -> None
        """
        Set the URI of the storage destination for uploaded model weight files.
        Supported storage destinations include S3, Google Cloud Storage), and file locations.

        Using this method, files uploads are separate and then a link to each is stored in the model object.

        .. note::
           For storage requiring credentials, the credentials are stored in the ClearML configuration file,
           ``~/clearml.conf``.

        :param str uri: The URI of the upload storage destination.

            For example:

            - ``s3://bucket/directory/``
            - ``file:///tmp/debug/``

        :return bool: The status of whether the storage destination schema is supported.

            - ``True`` - The storage destination scheme is supported.
            - ``False`` - The storage destination scheme is not supported.
        """
        if not uri:
            return

        # Test if we can update the model.
        self._validate_update()

        # Create the storage helper
        storage = StorageHelper.get(uri)

        # Verify that we can upload to this destination
        try:
            uri = storage.verify_upload(folder_uri=uri)
        except Exception:
            raise ValueError("Could not set destination uri to: %s [Check write permissions]" % uri)

        # store default uri
        self._get_base_model().upload_storage_uri = uri

    def update_weights(
        self,
        weights_filename=None,  # type: Optional[str]
        upload_uri=None,  # type: Optional[str]
        target_filename=None,  # type: Optional[str]
        auto_delete_file=True,  # type: bool
        register_uri=None,  # type: Optional[str]
        iteration=None,  # type: Optional[int]
        update_comment=True,  # type: bool
        is_package=False,  # type: bool
    ):
        # type: (...) -> str
        """
        Update the model weights from a locally stored model filename.

        .. note::
           Uploading the model is a background process. A call to this method returns immediately.

        :param str weights_filename: The name of the locally stored weights file to upload.
            Specify ``weights_filename`` or ``register_uri``, but not both.
        :param str upload_uri: The URI of the storage destination for model weights upload. The default value
            is the previously used URI. (Optional)
        :param str target_filename: The newly created filename in the storage destination location. The default value
            is the ``weights_filename`` value. (Optional)
        :param bool auto_delete_file: Delete the temporary file after uploading (Optional)

            - ``True`` - Delete (Default)
            - ``False`` - Do not delete

        :param str register_uri: The URI of an already uploaded weights file. The URI must be valid. Specify
            ``register_uri`` or ``weights_filename``, but not both.
        :param int iteration: The iteration number.
        :param bool update_comment: Update the model comment with the local weights file name (to maintain
            provenance) (Optional)

            - ``True`` - Update model comment (Default)
            - ``False`` - Do not update
        :param bool is_package: Mark the weights file as compressed package, usually a zip file.

        :return: The uploaded URI.
        """

        def delete_previous_weights_file(filename=weights_filename):
            try:
                if filename:
                    os.remove(filename)
            except OSError:
                self._log.debug('Failed removing temporary file %s' % filename)

        # test if we can update the model
        if self.id and self.published:
            raise ValueError('Model is published and cannot be changed')

        if (not weights_filename and not register_uri) or (weights_filename and register_uri):
            raise ValueError('Model update must have either local weights file to upload, '
                             'or pre-uploaded register_uri, never both')

        # only upload if we are connected to a task
        if not self._task:
            raise Exception('Missing a task for this model')

        if weights_filename is not None:
            # Check if weights_filename is a folder, is package upload
            if Path(weights_filename).is_dir():
                return self.update_weights_package(
                    weights_path=weights_filename,
                    upload_uri=upload_uri,
                    target_filename=target_filename or Path(weights_filename).name,
                    auto_delete_file=auto_delete_file,
                    iteration=iteration)

            # make sure we delete the previous file, if it exists
            if self._model_local_filename != weights_filename:
                delete_previous_weights_file(self._model_local_filename)
            # store temp filename for deletion next time, if needed
            if auto_delete_file:
                self._model_local_filename = weights_filename

        # make sure the created model is updated:
        out_model_file_name = target_filename or weights_filename or register_uri
        name = Path(out_model_file_name).stem if out_model_file_name else (self._task_connect_name or "Output Model")
        model = self._get_force_base_model(task_model_entry=name)
        if not model:
            raise ValueError('Failed creating internal output model')

        # select the correct file extension based on the framework,
        # or update the framework based on the file extension
        # noinspection PyProtectedMember
        framework, file_ext = Framework._get_file_ext(
            framework=self._get_model_data().framework,
            filename=target_filename or weights_filename or register_uri
        )

        if weights_filename:
            target_filename = target_filename or Path(weights_filename).name
            if not target_filename.lower().endswith(file_ext):
                target_filename += file_ext

        # set target uri for upload (if specified)
        if upload_uri:
            self.set_upload_destination(upload_uri)

        # let us know the iteration number, we put it in the comment section for now.
        if update_comment:
            comment = self.comment or ''
            iteration_msg = 'snapshot {} stored'.format(weights_filename or register_uri)
            if not comment.startswith('\n'):
                comment = '\n' + comment
            comment = iteration_msg + comment
        else:
            comment = None

        # if we have no output destination, just register the local model file
        if weights_filename and not self.upload_storage_uri and not self._task.storage_uri:
            register_uri = weights_filename
            weights_filename = None
            auto_delete_file = False
            self._log.info('No output storage destination defined, registering local model %s' % register_uri)

        # start the upload
        if weights_filename:
            if not model.upload_storage_uri:
                self.set_upload_destination(self.upload_storage_uri or self._task.storage_uri)

            output_uri = model.update_and_upload(
                model_file=weights_filename,
                task_id=self._task.id,
                async_enable=True,
                target_filename=target_filename,
                framework=self.framework or framework,
                comment=comment,
                cb=delete_previous_weights_file if auto_delete_file else None,
                iteration=iteration or self._task.get_last_iteration(),
            )
        elif register_uri:
            register_uri = StorageHelper.conform_url(register_uri)
            output_uri = model.update(uri=register_uri, task_id=self._task.id, framework=framework, comment=comment)
        else:
            output_uri = None

        self._last_uploaded_url = output_uri

        if is_package:
            self._set_package_tag()

        return output_uri

    def update_weights_package(
        self,
        weights_filenames=None,  # type: Optional[Sequence[str]]
        weights_path=None,  # type: Optional[str]
        upload_uri=None,  # type: Optional[str]
        target_filename=None,  # type: Optional[str]
        auto_delete_file=True,  # type: bool
        iteration=None  # type: Optional[int]
    ):
        # type: (...) -> str
        """
        Update the model weights from locally stored model files, or from directory containing multiple files.

        .. note::
           Uploading the model weights is a background process. A call to this method returns immediately.

        :param weights_filenames: The file names of the locally stored model files. Specify ``weights_filenames``,
            or ``weights_path``, but not both.
        :type weights_filenames: list(str)
        :param weights_path: The directory path to a package. All the files in the directory will be uploaded.
            Specify ``weights_path`` or ``weights_filenames``, but not both.
        :type weights_path: str
        :param str upload_uri: The URI of the storage destination for the model weights upload. The default
            is the previously used URI. (Optional)
        :param str target_filename: The newly created filename in the storage destination URI location. The default
            is the value specified in the ``weights_filename`` parameter.  (Optional)
        :param bool auto_delete_file: Delete temporary file after uploading  (Optional)

            - ``True`` - Delete (Default)
            - ``False`` - Do not delete

        :param int iteration: The iteration number.

        :return: The uploaded URI for the weights package.
        """
        # create list of files
        if (not weights_filenames and not weights_path) or (weights_filenames and weights_path):
            raise ValueError('Model update weights package should get either '
                             'directory path to pack or a list of files')

        if not weights_filenames:
            weights_filenames = list(map(six.text_type, Path(weights_path).rglob('*')))
        elif weights_filenames and len(weights_filenames) > 1:
            weights_path = get_common_path(weights_filenames)

        # create packed model from all the files
        fd, zip_file = mkstemp(prefix='model_package.', suffix='.zip')
        try:
            with zipfile.ZipFile(zip_file, 'w', allowZip64=True, compression=zipfile.ZIP_STORED) as zf:
                for filename in weights_filenames:
                    relative_file_name = Path(filename).name if not weights_path else \
                        Path(filename).absolute().relative_to(Path(weights_path).absolute()).as_posix()
                    zf.write(filename, arcname=relative_file_name)
        finally:
            os.close(fd)

        # now we can delete the files (or path if provided)
        if auto_delete_file:
            def safe_remove(path, is_dir=False):
                try:
                    (os.rmdir if is_dir else os.remove)(path)
                except OSError:
                    self._log.info('Failed removing temporary {}'.format(path))

            for filename in weights_filenames:
                safe_remove(filename)
            if weights_path:
                safe_remove(weights_path, is_dir=True)

        if target_filename and not target_filename.lower().endswith('.zip'):
            target_filename += '.zip'

        # and now we should upload the file, always delete the temporary zip file
        comment = self.comment or ''
        iteration_msg = 'snapshot {} stored'.format(str(weights_filenames))
        if not comment.startswith('\n'):
            comment = '\n' + comment
        comment = iteration_msg + comment
        self.comment = comment
        uploaded_uri = self.update_weights(weights_filename=zip_file, auto_delete_file=True, upload_uri=upload_uri,
                                           target_filename=target_filename or 'model_package.zip',
                                           iteration=iteration, update_comment=False)
        # set the model tag (by now we should have a model object) so we know we have packaged file
        self._set_package_tag()
        return uploaded_uri

    def update_design(self, config_text=None, config_dict=None):
        # type: (Optional[str], Optional[dict]) -> bool
        """
        Update the model configuration. Store a blob of text for custom usage.

        .. note::
           This method's behavior is lazy. The design update is only forced when the weights
           are updated.

        :param config_text: The configuration as a string. This is usually the content of a configuration
            dictionary file. Specify ``config_text`` or ``config_dict``, but not both.
        :type config_text: unconstrained text string
        :param dict config_dict: The configuration as a dictionary. Specify ``config_text`` or ``config_dict``,
            but not both.

        :return: True, update successful. False, update not successful.
        """
        if not self._validate_update():
            return False

        config_text = self._resolve_config(config_text=config_text, config_dict=config_dict)

        if self._task and not self._task.get_model_config_text():
            self._task.set_model_config(config_text=config_text)

        if self.id:
            # update the model object (this will happen if we resumed a training task)
            result = self._get_force_base_model().edit(design=config_text)
        else:
            # noinspection PyProtectedMember
            self._floating_data.design = _Model._wrap_design(config_text)
            result = Waitable()

        # you can wait on this object
        return result

    def update_labels(self, labels):
        # type: (Mapping[str, int]) -> Optional[Waitable]
        """
        Update the label enumeration.

        :param dict labels: The label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        :return:
        """
        validate_dict(labels, key_types=six.string_types, value_types=six.integer_types, desc='label enumeration')

        if not self._validate_update():
            return

        if self._task:
            self._task.set_model_label_enumeration(labels)

        if self.id:
            # update the model object (this will happen if we resumed a training task)
            result = self._get_force_base_model().edit(labels=labels)
        else:
            self._floating_data.labels = labels
            result = Waitable()

        # you can wait on this object
        return result

    @classmethod
    def wait_for_uploads(cls, timeout=None, max_num_uploads=None):
        # type: (Optional[float], Optional[int]) -> None
        """
        Wait for any pending or in-progress model uploads to complete. If no uploads are pending or in-progress,
        then the ``wait_for_uploads`` returns immediately.

        :param float timeout: The timeout interval to wait for uploads (seconds). (Optional).
        :param int max_num_uploads: The maximum number of uploads to wait for. (Optional).
        """
        _Model.wait_for_results(timeout=timeout, max_num_uploads=max_num_uploads)

    @classmethod
    def set_default_upload_uri(cls, output_uri):
        # type: (Optional[str]) -> None
        """
        Set the default upload uri for all OutputModels

        :param output_uri: URL for uploading models. examples:
            https://demofiles.demo.clear.ml, s3://bucket/, gs://bucket/, azure://bucket/, file:///mnt/shared/nfs
        """
        cls._default_output_uri = str(output_uri) if output_uri else None

    def _get_force_base_model(self, model_name=None, task_model_entry=None):
        if self._base_model:
            return self._base_model

        # create a new model from the task
        # noinspection PyProtectedMember
        self._base_model = self._task._get_output_model(model_id=None)
        # update the model from the task inputs
        labels = self._task.get_labels_enumeration()
        # noinspection PyProtectedMember
        config_text = self._task._get_model_config_text()
        model_name = model_name or self._floating_data.name or self._task.name
        task_model_entry = task_model_entry or self._task_connect_name or Path(self._get_model_data().uri).stem
        parent = self._task.input_models_id.get(task_model_entry)
        self._base_model.update(
            labels=self._floating_data.labels or labels,
            design=self._floating_data.design or config_text,
            task_id=self._task.id,
            project_id=self._task.project,
            parent_id=parent,
            name=model_name,
            comment=self._floating_data.comment,
            tags=self._floating_data.tags,
            framework=self._floating_data.framework,
            upload_storage_uri=self._floating_data.upload_storage_uri
        )

        # remove model floating change set, by now they should have matched the task.
        self._floating_data = None

        # now we have to update the creator task so it points to us
        if str(self._task.status) not in (
                str(self._task.TaskStatusEnum.created), str(self._task.TaskStatusEnum.in_progress)):
            self._log.warning('Could not update last created model in Task {}, '
                              'Task status \'{}\' cannot be updated'.format(self._task.id, self._task.status))
        else:
            self._base_model.update_for_task(
                task_id=self._task.id, model_id=self.id, type_="output", name=task_model_entry)

        return self._base_model

    def _get_base_model(self):
        if self._floating_data:
            return self._floating_data
        return self._get_force_base_model()

    def _get_model_data(self):
        if self._base_model:
            return self._base_model.data
        return self._floating_data

    def _validate_update(self):
        # test if we can update the model
        if self.id and self.published:
            raise ValueError('Model is published and cannot be changed')

        return True

    def _get_last_uploaded_filename(self):
        if not self._last_uploaded_url and not self.url:
            return None
        return Path(self._last_uploaded_url or self.url).name


class Waitable(object):
    def wait(self, *_, **__):
        return True


# clearml/task.py

import atexit
import json
import os
import shutil
import signal
import sys
import threading
import time
from argparse import ArgumentParser
from logging import getLogger
from operator import attrgetter
from tempfile import mkstemp, mkdtemp
from zipfile import ZipFile, ZIP_DEFLATED

try:
    # noinspection PyCompatibility
    from collections.abc import Sequence as CollectionsSequence
except ImportError:
    from collections import Sequence as CollectionsSequence

from typing import Optional, Union, Mapping, Sequence, Any, Dict, Iterable, TYPE_CHECKING, Callable, Tuple, List

import psutil
import six
from pathlib2 import Path

from .backend_config.defs import get_active_config_file, get_config_file
from .backend_api.services import tasks, projects
from .backend_api.session.session import (
    Session, ENV_ACCESS_KEY, ENV_SECRET_KEY, ENV_HOST, ENV_WEB_HOST, ENV_FILES_HOST, )
from .backend_api.session.defs import ENV_DEFERRED_TASK_INIT
from .backend_interface.metrics import Metrics
from .backend_interface.model import Model as BackendModel
from .backend_interface.task import Task as _Task
from .backend_interface.task.log import TaskHandler
from .backend_interface.task.development.worker import DevWorker
from .backend_interface.task.repo import ScriptInfo
from .backend_interface.task.models import TaskModels
from .backend_interface.util import (
    get_single_result,
    exact_match_regex,
    make_message,
    mutually_exclusive,
    get_queue_id,
    get_num_enqueued_tasks,
    get_or_create_project,
)
from .binding.absl_bind import PatchAbsl
from .binding.artifacts import Artifacts, Artifact
from .binding.environ_bind import EnvironmentBind, PatchOsFork
from .binding.frameworks.fastai_bind import PatchFastai
from .binding.frameworks.lightgbm_bind import PatchLIGHTgbmModelIO
from .binding.frameworks.pytorch_bind import PatchPyTorchModelIO
from .binding.frameworks.tensorflow_bind import TensorflowBinding
from .binding.frameworks.xgboost_bind import PatchXGBoostModelIO
from .binding.frameworks.catboost_bind import PatchCatBoostModelIO
from .binding.frameworks.megengine_bind import PatchMegEngineModelIO
from .binding.joblib_bind import PatchedJoblib
from .binding.matplotlib_bind import PatchedMatplotlib
from .binding.hydra_bind import PatchHydra
from .binding.click_bind import PatchClick
from .binding.fire_bind import PatchFire
from .binding.jsonargs_bind import PatchJsonArgParse
from .binding.frameworks import WeightsFileHandler
from .config import (
    config, DEV_TASK_NO_REUSE, get_is_master_node, DEBUG_SIMULATE_REMOTE_TASK, DEV_DEFAULT_OUTPUT_URI,
    deferred_config, TASK_SET_ITERATION_OFFSET, )
from .config import running_remotely, get_remote_task_id
from .config.cache import SessionCache
from .debugging.log import LoggerRoot
from .errors import UsageError
from .logger import Logger
from .model import Model, InputModel, OutputModel, Framework
from .task_parameters import TaskParameters
from .utilities.config import verify_basic_value
from .binding.args import (
    argparser_parseargs_called, get_argparser_last_args,
    argparser_update_currenttask, )
from .utilities.dicts import ReadOnlyDict, merge_dicts
from .utilities.proxy_object import (
    ProxyDictPreWrite, ProxyDictPostWrite, flatten_dictionary,
    nested_from_flat_dictionary, naive_nested_from_flat_dictionary, )
from .utilities.resource_monitor import ResourceMonitor
from .utilities.seed import make_deterministic
from .utilities.lowlevel.threads import get_current_thread_id
from .utilities.process.mp import BackgroundMonitor, leave_process
from .utilities.matching import matches_any_wildcard
from .utilities.parallel import FutureTaskCaller
# noinspection PyProtectedMember
from .backend_interface.task.args import _Arguments

if TYPE_CHECKING:
    import pandas
    import numpy
    from PIL import Image


class Task(_Task):
    """
    The ``Task`` class is a code template for a Task object which, together with its connected experiment components,
    represents the current running experiment. These connected components include hyperparameters, loggers,
    configuration, label enumeration, models, and other artifacts.

    The term "main execution Task" refers to the Task context for current running experiment. Python experiment scripts
    can create one, and only one, main execution Task. It is traceable, and after a script runs and ClearML stores
    the Task in the **ClearML Server** (backend), it is modifiable, reproducible, executable by a worker, and you
    can duplicate it for further experimentation.

    The ``Task`` class and its methods allow you to create and manage experiments, as well as perform
    advanced experimentation functions, such as autoML.

    .. warning::
        Do not construct Task objects directly. Use one of the methods listed below to create experiments or
        reference existing experiments.
        Do not define `CLEARML_TASK_*` and `CLEARML_PROC_*` OS environments, they are used internally
        for bookkeeping between processes and agents.

    For detailed information about creating Task objects, see the following methods:

    - Create a new reproducible Task - :meth:`Task.init`

    .. important::
        In some cases, ``Task.init`` may return a Task object which is already stored in **ClearML Server** (already
        initialized), instead of creating a new Task. For a detailed explanation of those cases, see the ``Task.init``
        method.

    - Manually create a new Task (no auto-logging will apply) - :meth:`Task.create`
    - Get the current running Task - :meth:`Task.current_task`
    - Get another (different) Task - :meth:`Task.get_task`

    .. note::
        The **ClearML** documentation often refers to a Task as, "Task (experiment)".

        "Task" refers to the class in the ClearML Python Client Package, the object in your Python experiment script,
        and the entity with which **ClearML Server** and **ClearML Agent** work.

        "Experiment" refers to your deep learning solution, including its connected components, inputs, and outputs,
        and is the experiment you can view, analyze, compare, modify, duplicate, and manage using the ClearML
        **Web-App** (UI).

        Therefore, a "Task" is effectively an "experiment", and "Task (experiment)" encompasses its usage throughout
        the ClearML.

        The exception to this Task behavior is sub-tasks (non-reproducible Tasks), which do not use the main execution
        Task. Creating a sub-task always creates a new Task with a new  Task ID.
    """

    TaskTypes = _Task.TaskTypes

    NotSet = object()

    __create_protection = object()
    __main_task = None  # type: Optional[Task]
    __exit_hook = None
    __forked_proc_main_pid = None
    __task_id_reuse_time_window_in_hours = deferred_config('development.task_reuse_time_window_in_hours', 24.0, float)
    __detect_repo_async = deferred_config('development.vcs_repo_detect_async', False)
    __default_output_uri = DEV_DEFAULT_OUTPUT_URI.get() or deferred_config('development.default_output_uri', None)

    class _ConnectedParametersType(object):
        argparse = "argument_parser"
        dictionary = "dictionary"
        task_parameters = "task_parameters"

        @classmethod
        def _options(cls):
            return {
                var for var, val in vars(cls).items()
                if isinstance(val, six.string_types)
            }

    def __init__(self, private=None, **kwargs):
        """
        .. warning::
            **Do not construct Task manually!**
            Please use :meth:`Task.init` or :meth:`Task.get_task`
        """
        if private is not Task.__create_protection:
            raise UsageError(
                'Task object cannot be instantiated externally, use Task.current_task() or Task.get_task(...)')
        self._repo_detect_lock = threading.RLock()

        super(Task, self).__init__(**kwargs)
        self._arguments = _Arguments(self)
        self._logger = None
        self._connected_output_model = None
        self._dev_worker = None
        self._connected_parameter_type = None
        self._detect_repo_async_thread = None
        self._resource_monitor = None
        self._calling_filename = None
        self._remote_functions_generated = {}
        # register atexit, so that we mark the task as stopped
        self._at_exit_called = False

    @classmethod
    def current_task(cls):
        # type: () -> Task
        """
        Get the current running Task (experiment). This is the main execution Task (task context) returned as a Task
        object.

        :return: The current running Task (experiment).
        """
        # check if we have no main Task, but the main process created one.
        if not cls.__main_task and cls.__get_master_id_task_id():
            # initialize the Task, connect to stdout
            cls.init()
        # return main Task
        return cls.__main_task

    @classmethod
    def init(
            cls,
            project_name=None,  # type: Optional[str]
            task_name=None,  # type: Optional[str]
            task_type=TaskTypes.training,  # type: Task.TaskTypes
            tags=None,  # type: Optional[Sequence[str]]
            reuse_last_task_id=True,  # type: Union[bool, str]
            continue_last_task=False,  # type: Union[bool, str, int]
            output_uri=None,  # type: Optional[Union[str, bool]]
            auto_connect_arg_parser=True,  # type: Union[bool, Mapping[str, bool]]
            auto_connect_frameworks=True,  # type: Union[bool, Mapping[str, Union[bool, str, list]]]
            auto_resource_monitoring=True,  # type: bool
            auto_connect_streams=True,  # type: Union[bool, Mapping[str, bool]]
            deferred_init=False,  # type: bool
    ):
        # type: (...) -> Task
        """
        Creates a new Task (experiment) if:

        - The Task never ran before. No Task with the same ``task_name`` and ``project_name`` is stored in
          **ClearML Server**.
        - The Task has run before (the same ``task_name`` and ``project_name``), and (a) it stored models and / or
          artifacts, or (b) its status is Published , or (c) it is Archived.
        - A new Task is forced by calling ``Task.init`` with ``reuse_last_task_id=False``.

        Otherwise, the already initialized Task object for the same ``task_name`` and ``project_name`` is returned,
        or, when being executed remotely on a clearml-agent, the task returned is the existing task from the backend.

        .. note::
            To reference another Task, instead of initializing the same Task more than once, call
            :meth:`Task.get_task`. For example, to "share" the same experiment in more than one script,
            call ``Task.get_task``. See the ``Task.get_task`` method for an example.

        For example:
        The first time the following code runs, it will create a new Task. The status will be Completed.

        .. code-block:: py

            from clearml import Task
            task = Task.init('myProject', 'myTask')

        If this code runs again, it will not create a new Task. It does not store a model or artifact,
        it is not Published (its status Completed) , it was not Archived, and a new Task is not forced.

        If the Task is Published or Archived, and run again, it will create a new Task with a new Task ID.

        The following code will create a new Task every time it runs, because it stores an artifact.

        .. code-block:: py

            task = Task.init('myProject', 'myOtherTask')

            d = {'a': '1'}
            task.upload_artifact('myArtifact', d)

        :param str project_name: The name of the project in which the experiment will be created. If the project does
            not exist, it is created. If ``project_name`` is ``None``, the repository name is used. (Optional)
        :param str task_name: The name of Task (experiment). If ``task_name`` is ``None``, the Python experiment
            script's file name is used. (Optional)
        :param TaskTypes task_type: The task type.

            Valid task types:

            - ``TaskTypes.training`` (default)
            - ``TaskTypes.testing``
            - ``TaskTypes.inference``
            - ``TaskTypes.data_processing``
            - ``TaskTypes.application``
            - ``TaskTypes.monitor``
            - ``TaskTypes.controller``
            - ``TaskTypes.optimizer``
            - ``TaskTypes.service``
            - ``TaskTypes.qc``
            - ``TaskTypes.custom``

        :param tags: Add a list of tags (str) to the created Task. For example: tags=['512x512', 'yolov3']
        :param bool reuse_last_task_id: Force a new Task (experiment) with a previously used Task ID,
            and the same project and Task name.

            .. note::
               If the previously executed Task has artifacts or models, it will not be reused (overwritten)
               and a new Task will be created.
               When a Task is reused, the previous execution outputs are deleted, including console outputs and logs.

            The values are:

            - ``True`` - Reuse the last  Task ID. (default)
            - ``False`` - Force a new Task (experiment).
            - A string - You can also specify a Task ID (string) to be reused,
                instead of the cached ID based on the project/name combination.

        :param bool continue_last_task: Continue the execution of a previously executed Task (experiment)

            .. note::
                When continuing the executing of a previously executed Task,
                all previous artifacts / models/ logs are intact.
                New logs will continue iteration/step based on the previous-execution maximum iteration value.
                For example:
                The last train/loss scalar reported was iteration 100, the next report will be iteration 101.

            The values are:

            - ``True`` - Continue the last Task ID.
                specified explicitly by reuse_last_task_id or implicitly with the same logic as reuse_last_task_id
            - ``False`` - Overwrite the execution of previous Task  (default).
            - A string - You can also specify a Task ID (string) to be continued.
                This is equivalent to `continue_last_task=True` and `reuse_last_task_id=a_task_id_string`.
            - An integer - Specify initial iteration offset (override the auto automatic last_iteration_offset)
                Pass 0, to disable the automatic last_iteration_offset or specify a different initial offset
                You can specify a Task ID to be used with `reuse_last_task_id='task_id_here'`

        :param str output_uri: The default location for output models and other artifacts.
            If True is passed, the default files_server will be used for model storage.
            In the default location, ClearML creates a subfolder for the output.
            The subfolder structure is the following:
            <output destination name> / <project name> / <task name>.<Task ID>

            The following are examples of ``output_uri`` values for the supported locations:

            - A shared folder: ``/mnt/share/folder``
            - S3: ``s3://bucket/folder``
            - Google Cloud Storage: ``gs://bucket-name/folder``
            - Azure Storage: ``azure://company.blob.core.windows.net/folder/``
            - Default file server: True

            .. important::

               For cloud storage, you must install the **ClearML** package for your cloud storage type,
               and then configure your storage credentials. For detailed information, see
               `ClearML Python Client Extras <./references/clearml_extras_storage/>`_ in the "ClearML Python Client
               Reference" section.

        :param auto_connect_arg_parser: Automatically connect an argparse object to the Task. Supported argument
            parsers packages are: argparse, click, python-fire, jsonargparse.

            The values are:

            - ``True`` - Automatically connect. (default)
            - ``False`` - Do not automatically connect.
            - A dictionary - In addition to a boolean, you can use a dictionary for fined grained control of connected
                arguments. The dictionary keys are argparse variable names and the values are booleans.
                The ``False`` value excludes the specified argument from the Task's parameter section.
                Keys missing from the dictionary default to ``True``, you can change it to be ``False`` by adding
                ``*`` key as ``False`` to the dictionary.
                An empty dictionary defaults to ``False``.

            For example:

            .. code-block:: py

               auto_connect_arg_parser={'do_not_include_me': False, }

            .. code-block:: py

               auto_connect_arg_parser={"only_include_me": True, "*": False}

            .. note::
               To manually connect an argparse, use :meth:`Task.connect`.

        :param auto_connect_frameworks: Automatically connect frameworks This includes patching MatplotLib, XGBoost,
            scikit-learn, Keras callbacks, and TensorBoard/X to serialize plots, graphs, and the model location to
            the **ClearML Server** (backend), in addition to original output destination.

            The values are:

            - ``True`` - Automatically connect (default)
            - ``False`` - Do not automatically connect
            - A dictionary - In addition to a boolean, you can use a dictionary for fined grained control of connected
                frameworks. The dictionary keys are frameworks and the values are booleans, other dictionaries used for
                finer control or wildcard strings.
                In case of wildcard strings, the local path of a model file has to match at least one wildcard to be
                saved/loaded by ClearML. Example:
                    {'pytorch' : '*.pt', 'tensorflow': ['*.h5', '*']}
                Keys missing from the dictionary default to ``True``, and an empty dictionary defaults to ``False``.
                Supported keys for finer control:
                    {'tensorboard': {'report_hparams': bool}}  # whether to report TensorBoard hyperparameters

            For example:

            .. code-block:: py

               auto_connect_frameworks={
                   'matplotlib': True, 'tensorflow': ['*.hdf5, 'something_else*], 'tensorboard': True,
                   'pytorch': ['*.pt'], 'xgboost': True, 'scikit': True, 'fastai': True,
                   'lightgbm': True, 'hydra': True, 'detect_repository': True, 'tfdefines': True,
                   'joblib': True, 'megengine': True, 'catboost': True
               }

            .. code-block:: py
                auto_connect_frameworks={'tensorboard': {'report_hparams': False}}

        :param bool auto_resource_monitoring: Automatically create machine resource monitoring plots
            These plots appear in the **ClearML Web-App (UI)**, **RESULTS** tab, **SCALARS** sub-tab,
            with a title of **:resource monitor:**.

            The values are:

            - ``True`` - Automatically create resource monitoring plots. (default)
            - ``False`` - Do not automatically create.
            - Class Type - Create ResourceMonitor object of the specified class type.

        :param auto_connect_streams: Control the automatic logging of stdout and stderr

            The values are:

            - ``True`` - Automatically connect (default)
            -  ``False`` - Do not automatically connect
            - A dictionary - In addition to a boolean, you can use a dictionary for fined grained control of stdout and
                stderr. The dictionary keys are 'stdout' , 'stderr' and 'logging', the values are booleans.
                Keys missing from the dictionary default to ``False``, and an empty dictionary defaults to ``False``.
                Notice, the default behaviour is logging stdout/stderr the
                `logging` module is logged as a by product of the stderr logging

            For example:

            .. code-block:: py

               auto_connect_streams={'stdout': True, 'stderr': True, 'logging': False}

        :param deferred_init: (default: False) Wait for Task to be fully initialized (regular behaviour).

            ** BETA feature! use with care **

            If set to True, `Task.init` function returns immediately and all initialization / communication
            to the clearml-server is running in a background thread. The returned object is
            a full proxy to the regular Task object, hence everything will be working as expected.
            Default behaviour can be controlled with:
                `CLEARML_DEFERRED_TASK_INIT=1`

            Notes:

            - Any access to the returned proxy `Task` object will essentially wait for the `Task.init`
                to be completed. For example: `print(task.name)` will wait for `Task.init` to complete in the
                background and then return the `name` property of the task original object
            - Before `Task.init` completes in the background, auto-magic logging
                (console/metric) might be missed
            - If running via an agent, this argument is ignored,
                and Task init is called synchronously (default)

        :return: The main execution Task (Task context)
        """

        def verify_defaults_match():
            validate = [
                ('project name', project_name, cls.__main_task.get_project_name()),
                ('task name', task_name, cls.__main_task.name),
                ('task type', str(task_type) if task_type else task_type, str(cls.__main_task.task_type)),
            ]

            for field, default, current in validate:
                if default is not None and default != current:
                    raise UsageError(
                        "Current task already created "
                        "and requested {field} '{default}' does not match current {field} '{current}'. "
                        "If you wish to create additional tasks use `Task.create`, "
                        "or close the current task with `task.close()` before calling `Task.init(...)`".format(
                            field=field,
                            default=default,
                            current=current,
                        )
                    )

        # if deferred_init==0 this means this is the nested call that actually generates the Task.init
        if cls.__main_task is not None and deferred_init != 0:
            # if this is a subprocess, regardless of what the init was called for,
            # we have to fix the main task hooks and stdout bindings
            if cls.__forked_proc_main_pid != os.getpid() and cls.__is_subprocess():
                if task_type is None:
                    task_type = cls.__main_task.task_type
                # make sure we only do it once per process
                cls.__forked_proc_main_pid = os.getpid()
                # make sure we do not wait for the repo detect thread
                cls.__main_task._detect_repo_async_thread = None
                cls.__main_task._dev_worker = None
                cls.__main_task._resource_monitor = None
                # remove the logger from the previous process
                cls.__main_task.get_logger()
                # create a new logger (to catch stdout/err)
                cls.__main_task._logger = None
                cls.__main_task.__reporter = None
                # noinspection PyProtectedMember
                cls.__main_task._get_logger(auto_connect_streams=auto_connect_streams)
                cls.__main_task._artifacts_manager = Artifacts(cls.__main_task)
                # unregister signal hooks, they cause subprocess to hang
                # noinspection PyProtectedMember
                cls.__main_task.__register_at_exit(cls.__main_task._at_exit)
                # TODO: Check if the signal handler method is safe enough, for the time being, do not unhook
                # cls.__main_task.__register_at_exit(None, only_remove_signal_and_exception_hooks=True)

                # start all reporting threads
                BackgroundMonitor.start_all(task=cls.__main_task)

            if not running_remotely():
                verify_defaults_match()

            return cls.__main_task

        is_sub_process_task_id = None
        # check that we are not a child process, in that case do nothing.
        # we should not get here unless this is Windows/macOS platform, linux support fork
        if cls.__is_subprocess():
            class _TaskStub(object):
                def __call__(self, *args, **kwargs):
                    return self

                def __getattr__(self, attr):
                    return self

                def __setattr__(self, attr, val):
                    pass

            is_sub_process_task_id = cls.__get_master_id_task_id()
            # we could not find a task ID, revert to old stub behaviour
            if not is_sub_process_task_id:
                return _TaskStub()  # noqa

        elif running_remotely() and not get_is_master_node():
            # make sure we only do it once per process
            cls.__forked_proc_main_pid = os.getpid()
            # make sure everyone understands we should act as if we are a subprocess (fake pid 1)
            cls.__update_master_pid_task(pid=1, task=get_remote_task_id())
        else:
            # set us as master process (without task ID)
            cls.__update_master_pid_task()
            is_sub_process_task_id = None

        if task_type is None:
            # Backwards compatibility: if called from Task.current_task and task_type
            # was not specified, keep legacy default value of TaskTypes.training
            task_type = cls.TaskTypes.training
        elif isinstance(task_type, six.string_types):
            if task_type not in Task.TaskTypes.__members__:
                raise ValueError("Task type '{}' not supported, options are: {}".format(
                    task_type, Task.TaskTypes.__members__.keys()))
            task_type = Task.TaskTypes.__members__[str(task_type)]

        is_deferred = False
        try:
            if not running_remotely():
                # only allow if running locally and creating the first Task
                # otherwise we ignore and perform in order
                if deferred_init != 0 and ENV_DEFERRED_TASK_INIT.get():
                    deferred_init = True
                if not is_sub_process_task_id and deferred_init:
                    def completed_cb(x):
                        Task.__main_task = x

                    getLogger().warning("ClearML initializing Task in the background")

                    task = FutureTaskCaller(
                        func=cls.init,
                        func_cb=completed_cb,
                        override_cls=cls,
                        project_name=project_name,
                        task_name=task_name,
                        tags=tags,
                        reuse_last_task_id=reuse_last_task_id,
                        continue_last_task=continue_last_task,
                        output_uri=output_uri,
                        auto_connect_arg_parser=auto_connect_arg_parser,
                        auto_connect_frameworks=auto_connect_frameworks,
                        auto_resource_monitoring=auto_resource_monitoring,
                        auto_connect_streams=auto_connect_streams,
                        deferred_init=0,  # notice we use it as a flag to mark the nested call
                    )
                    is_deferred = True
                    # mark as temp master
                    cls.__update_master_pid_task()
                # if this is the main process, create the task
                elif not is_sub_process_task_id:
                    task = cls._create_dev_task(
                        default_project_name=project_name,
                        default_task_name=task_name,
                        default_task_type=task_type,
                        tags=tags,
                        reuse_last_task_id=reuse_last_task_id,
                        continue_last_task=continue_last_task,
                        detect_repo=False if (
                                isinstance(auto_connect_frameworks, dict) and
                                not auto_connect_frameworks.get('detect_repository', True)) else True,
                        auto_connect_streams=auto_connect_streams,
                    )
                    # set defaults
                    if cls._offline_mode:
                        task.output_uri = None
                    elif output_uri:
                        task.output_uri = output_uri
                    elif cls.__default_output_uri:
                        task.output_uri = cls.__default_output_uri
                    # store new task ID
                    cls.__update_master_pid_task(task=task)
                else:
                    # subprocess should get back the task info
                    task = cls.get_task(task_id=is_sub_process_task_id)
            else:
                # if this is the main process, create the task
                if not is_sub_process_task_id:
                    task = cls(
                        private=cls.__create_protection,
                        task_id=get_remote_task_id(),
                        log_to_backend=False,
                    )
                    if cls.__default_output_uri and not task.output_uri:
                        task.output_uri = cls.__default_output_uri
                    # store new task ID
                    cls.__update_master_pid_task(task=task)
                    # make sure we are started
                    task.started(ignore_errors=True)
                    # continue last iteration if we had any
                    if task.data.last_iteration:
                        task.set_initial_iteration(int(task.data.last_iteration) + 1)
                else:
                    # subprocess should get back the task info
                    task = cls.get_task(task_id=is_sub_process_task_id)
        except Exception:
            raise
        else:
            Task.__main_task = task

            # register at exist only on the real (none deferred) Task
            if not is_deferred:
                # register the main task for at exit hooks (there should only be one)
                task.__register_at_exit(task._at_exit)

            # always patch OS forking because of ProcessPool and the alike
            PatchOsFork.patch_fork(task)
            if auto_connect_frameworks:
                def should_connect(*keys):
                    """
                    Evaluates value of auto_connect_frameworks[keys[0]]...[keys[-1]].
                    If at some point in the evaluation, the value of auto_connect_frameworks[keys[0]]...[keys[-1]]
                    is a bool, that value will be returned. If a dictionary is empty, it will be evaluated to False.
                    If a key will not be found in the current dictionary, True will be returned.
                    """
                    should_bind_framework = auto_connect_frameworks
                    for key in keys:
                        if not isinstance(should_bind_framework, dict):
                            return bool(should_bind_framework)
                        if should_bind_framework == {}:
                            return False
                        should_bind_framework = should_bind_framework.get(key, True)
                    return bool(should_bind_framework)

                if not is_deferred and should_connect("hydra"):
                    PatchHydra.update_current_task(task)
                if should_connect("scikit") and should_connect("joblib"):
                    PatchedJoblib.update_current_task(task)
                if should_connect("matplotlib"):
                    PatchedMatplotlib.update_current_task(task)
                if should_connect("tensorflow") or should_connect("tensorboard"):
                    # allow disabling tfdefines
                    if not is_deferred and should_connect("tfdefines"):
                        PatchAbsl.update_current_task(task)
                    TensorflowBinding.update_current_task(
                        task,
                        patch_reporting=should_connect("tensorboard"),
                        patch_model_io=should_connect("tensorflow"),
                        report_hparams=should_connect("tensorboard", "report_hparams"),
                    )
                if should_connect("pytorch"):
                    PatchPyTorchModelIO.update_current_task(task)
                if should_connect("megengine"):
                    PatchMegEngineModelIO.update_current_task(task)
                if should_connect("xgboost"):
                    PatchXGBoostModelIO.update_current_task(task)
                if should_connect("catboost"):
                    PatchCatBoostModelIO.update_current_task(task)
                if should_connect("fastai"):
                    PatchFastai.update_current_task(task)
                if should_connect("lightgbm"):
                    PatchLIGHTgbmModelIO.update_current_task(task)

                cls.__add_model_wildcards(auto_connect_frameworks)

            # if we are deferred, stop here (the rest we do in the actual init)
            if is_deferred:
                from .backend_interface.logger import StdStreamPatch
                # patch console outputs, we will keep them in memory until we complete the Task init
                # notice we do not load config defaults, as they are not threadsafe
                # we might also need to override them with the vault
                StdStreamPatch.patch_std_streams(
                    task.get_logger(),
                    connect_stdout=(
                        auto_connect_streams is True) or (
                            isinstance(auto_connect_streams, dict) and auto_connect_streams.get('stdout', False)
                    ),
                    connect_stderr=(
                        auto_connect_streams is True) or (
                            isinstance(auto_connect_streams, dict) and auto_connect_streams.get('stderr', False)
                    ),
                    load_config_defaults=False,
                )
                return task  # noqa

            if auto_resource_monitoring and not is_sub_process_task_id:
                resource_monitor_cls = auto_resource_monitoring \
                    if isinstance(auto_resource_monitoring, six.class_types) else ResourceMonitor
                task._resource_monitor = resource_monitor_cls(
                    task, report_mem_used_per_process=not config.get(
                        'development.worker.report_global_mem_used', False))
                task._resource_monitor.start()

            # make sure all random generators are initialized with new seed
            random_seed = task.get_random_seed()
            if random_seed is not None:
                make_deterministic(random_seed)
            task._set_random_seed_used(random_seed)

            if auto_connect_arg_parser:
                EnvironmentBind.update_current_task(task)

                PatchJsonArgParse.update_current_task(task)

                # Patch ArgParser to be aware of the current task
                argparser_update_currenttask(task)

                PatchClick.patch(task)
                PatchFire.patch(task)

                # set excluded arguments
                if isinstance(auto_connect_arg_parser, dict):
                    task._arguments.exclude_parser_args(auto_connect_arg_parser)

                # Check if parse args already called. If so, sync task parameters with parser
                if argparser_parseargs_called():
                    for parser, parsed_args in get_argparser_last_args():
                        task._connect_argparse(parser=parser, parsed_args=parsed_args)
            elif argparser_parseargs_called():
                # actually we have nothing to do, in remote running, the argparser will ignore
                # all non argparser parameters, only caveat if parameter connected with the same name
                # as the argparser this will be solved once sections are introduced to parameters
                pass

        # Make sure we start the logger, it will patch the main logging object and pipe all output
        # if we are running locally and using development mode worker, we will pipe all stdout to logger.
        # The logger will automatically take care of all patching (we just need to make sure to initialize it)
        logger = task._get_logger(auto_connect_streams=auto_connect_streams)
        # show the debug metrics page in the log, it is very convenient
        if not is_sub_process_task_id:
            if cls._offline_mode:
                logger.report_text('ClearML running in offline mode, session stored in {}'.format(
                    task.get_offline_mode_folder()))
            else:
                logger.report_text('ClearML results page: {}'.format(task.get_output_log_web_page()))
        # Make sure we start the dev worker if required, otherwise it will only be started when we write
        # something to the log.
        task._dev_mode_setup_worker()

        if (not task._reporter or not task._reporter.is_constructed()) and \
                is_sub_process_task_id and not cls._report_subprocess_enabled:
            task._setup_reporter()

        # start monitoring in background process or background threads
        # monitoring are: Resource monitoring and Dev Worker monitoring classes
        BackgroundMonitor.start_all(task=task)

        task.set_progress(0)
        return task

    @classmethod
    def create(
            cls,
            project_name=None,  # type: Optional[str]
            task_name=None,  # type: Optional[str]
            task_type=None,  # type: Optional[str]
            repo=None,  # type: Optional[str]
            branch=None,  # type: Optional[str]
            commit=None,  # type: Optional[str]
            script=None,  # type: Optional[str]
            working_directory=None,  # type: Optional[str]
            packages=None,  # type: Optional[Union[bool, Sequence[str]]]
            requirements_file=None,  # type: Optional[Union[str, Path]]
            docker=None,  # type: Optional[str]
            docker_args=None,  # type: Optional[str]
            docker_bash_setup_script=None,  # type: Optional[str]
            argparse_args=None,  # type: Optional[Sequence[Tuple[str, str]]]
            base_task_id=None,  # type: Optional[str]
            add_task_init_call=True,  # type: bool
    ):
        # type: (...) -> Task
        """
        Manually create and populate a new Task (experiment) in the system.
        If the code does not already contain a call to ``Task.init``, pass add_task_init_call=True,
        and the code will be patched in remote execution (i.e. when executed by `clearml-agent`

        .. note::
           This method **always** creates a new Task.
           Use :meth:`Task.init` method to automatically create and populate task for the running process.
           To reference an existing Task, call the  :meth:`Task.get_task` method .

        :param project_name: Set the project name for the task. Required if base_task_id is None.
        :param task_name: Set the name of the remote task. Required if base_task_id is None.
        :param task_type: Optional, The task type to be created. Supported values: 'training', 'testing', 'inference',
            'data_processing', 'application', 'monitor', 'controller', 'optimizer', 'service', 'qc', 'custom'
        :param repo: Remote URL for the repository to use, or path to local copy of the git repository
            Example: 'https://github.com/allegroai/clearml.git' or '~/project/repo'
        :param branch: Select specific repository branch/tag (implies the latest commit from the branch)
        :param commit: Select specific commit id to use (default: latest commit,
            or when used with local repository matching the local commit id)
        :param script: Specify the entry point script for the remote execution. When used in tandem with
            remote git repository the script should be a relative path inside the repository,
            for example: './source/train.py' . When used with local repository path it supports a
            direct path to a file inside the local repository itself, for example: '~/project/source/train.py'
        :param working_directory: Working directory to launch the script from. Default: repository root folder.
            Relative to repo root or local folder.
        :param packages: Manually specify a list of required packages. Example: ["tqdm>=2.1", "scikit-learn"]
            or `True` to automatically create requirements
            based on locally installed packages (repository must be local).
        :param requirements_file: Specify requirements.txt file to install when setting the session.
            If not provided, the requirements.txt from the repository will be used.
        :param docker: Select the docker image to be executed in by the remote session
        :param docker_args: Add docker arguments, pass a single string
        :param docker_bash_setup_script: Add bash script to be executed
            inside the docker before setting up the Task's environment
        :param argparse_args: Arguments to pass to the remote execution, list of string pairs (argument, value)
            Notice, only supported if the codebase itself uses argparse.ArgumentParser
        :param base_task_id: Use a pre-existing task in the system, instead of a local repo/script.
            Essentially clones an existing task and overrides arguments/requirements.
        :param add_task_init_call: If True, a 'Task.init()' call is added to the script entry point in remote execution.

        :return: The newly created Task (experiment)
        """
        if not project_name and not base_task_id:
            if not cls.__main_task:
                raise ValueError("Please provide project_name, no global task context found "
                                 "(Task.current_task hasn't been called)")
            project_name = cls.__main_task.get_project_name()
        from .backend_interface.task.populate import CreateAndPopulate
        manual_populate = CreateAndPopulate(
            project_name=project_name, task_name=task_name, task_type=task_type,
            repo=repo, branch=branch, commit=commit,
            script=script, working_directory=working_directory,
            packages=packages, requirements_file=requirements_file,
            docker=docker, docker_args=docker_args, docker_bash_setup_script=docker_bash_setup_script,
            base_task_id=base_task_id,
            add_task_init_call=add_task_init_call,
            raise_on_missing_entries=False,
        )
        task = manual_populate.create_task()
        if task and argparse_args:
            manual_populate.update_task_args(argparse_args)
            task.reload()

        return task

    @classmethod
    def get_task(
            cls,
            task_id=None,  # type: Optional[str]
            project_name=None,  # type: Optional[str]
            task_name=None,  # type: Optional[str]
            tags=None,  # type: Optional[Sequence[str]]
            allow_archived=True,  # type: bool
            task_filter=None  # type: Optional[dict]
    ):
        # type: (...) -> "Task"
        """
        Get a Task by Id, or project name / task name combination.

        For example:

        The following code demonstrates calling ``Task.get_task`` to report a scalar to another Task. The output
        of :meth:`.Logger.report_scalar` from testing is associated with the Task named ``training``. It allows
        training and testing to run concurrently, because they initialized different Tasks (see :meth:`Task.init`
        for information about initializing Tasks).

        The training script:

        .. code-block:: py

            # initialize the training Task
            task = Task.init('myProject', 'training')

            # do some training

        The testing script:

        .. code-block:: py

            # initialize the testing Task
            task = Task.init('myProject', 'testing')

            # get the training Task
            train_task = Task.get_task(project_name='myProject', task_name='training')

            # report metrics in the training Task
            for x in range(10):
                train_task.get_logger().report_scalar('title', 'series', value=x * 2, iteration=x)

        :param str task_id: The Id (system UUID) of the experiment to get.
            If specified, ``project_name`` and ``task_name`` are ignored.
        :param str project_name: The project name of the Task to get.
        :param str task_name: The name of the Task within ``project_name`` to get.
        :param list tags: Filter based on the requested list of tags (strings) (Task must have all the listed tags)
            To exclude a tag add "-" prefix to the tag. Example: ["best", "-debug"]
        :param bool allow_archived: Only applicable if *not* using specific ``task_id``,
            If True (default) allow to return archived Tasks, if False filter out archived Tasks
        :param bool task_filter: Only applicable if *not* using specific ``task_id``,
            Pass additional query filters, on top of project/name. See details in Task.get_tasks.

        :return: The Task specified by ID, or project name / experiment name combination.
        """
        return cls.__get_task(
            task_id=task_id, project_name=project_name, task_name=task_name, tags=tags,
            include_archived=allow_archived, task_filter=task_filter,
        )

    @classmethod
    def get_tasks(
            cls,
            task_ids=None,  # type: Optional[Sequence[str]]
            project_name=None,  # type: Optional[Union[Sequence[str],str]]
            task_name=None,  # type: Optional[str]
            tags=None,  # type: Optional[Sequence[str]]
            task_filter=None  # type: Optional[Dict]
    ):
        # type: (...) -> List["Task"]
        """
        Get a list of Tasks objects matching the queries/filters

        - A list of specific Task IDs.
        - Filter Tasks based on specific fields:
            project name (including partial match), task name (including partial match), tags
            Apply Additional advanced filtering with `task_filter`

        .. note::
            This function returns the most recent 500 tasks. If you wish to retrieve older tasks
            use ``Task.query_tasks()``

        :param list(str) task_ids: The Ids (system UUID) of experiments to get.
            If ``task_ids`` specified, then ``project_name`` and ``task_name`` are ignored.
        :param str project_name: The project name of the Tasks to get. To get the experiment
            in all projects, use the default value of ``None``. (Optional)
            Use a list of strings for multiple optional project names.
        :param str task_name: The full name or partial name of the Tasks to match within the specified
            ``project_name`` (or all projects if ``project_name`` is ``None``).
            This method supports regular expressions for name matching. (Optional)
            To match an exact task name (i.e. not partial matching),
            add ^/$ at the beginning/end of the string, for example: "^exact_task_name_here$"
        :param list(str) task_ids: list of unique task id string (if exists other parameters are ignored)
        :param str project_name: project name (str) the task belongs to (use None for all projects)
        :param str task_name: task name (str) in within the selected project
            Return any partial match of task_name, regular expressions matching is also supported
            If None is passed, returns all tasks within the project
        :param list tags: Filter based on the requested list of tags (strings) (Task must have all the listed tags)
            To exclude a tag add "-" prefix to the tag. Example: ["best", "-debug"]
        :param dict task_filter: filter and order Tasks. See service.tasks.GetAllRequest for details
            `parent`: (str) filter by parent task-id matching
            `search_text`: (str) free text search (in task fields comment/name/id)
            `status`: List[str] List of valid statuses
                (options are: "created", "queued", "in_progress", "stopped", "published", "publishing", "closed",
                "failed", "completed", "unknown")
            `type`: List[str] List of valid task type
                (options are: 'training', 'testing', 'inference', 'data_processing', 'application', 'monitor',
                'controller', 'optimizer', 'service', 'qc'. 'custom')
            `user`: List[str] Filter based on Task's user owner, provide list of valid user Ids.
            `order_by`: List[str] List of field names to order by. When search_text is used,
                Use '-' prefix to specify descending order. Optional, recommended when using page
                Example: order_by=['-last_update']
            `_all_`: dict(fields=[], pattern='')  Match string `pattern` (regular expression)
                appearing in All `fields`
                dict(fields=['script.repository'], pattern='github.com/user')
            `_any_`: dict(fields=[], pattern='')  Match string `pattern` (regular expression)
                appearing in Any of the `fields`
                dict(fields=['comment', 'name'], pattern='my comment')
            Examples:
                {'status': ['stopped'], 'order_by': ["-last_update"]}
                {'order_by'=['-last_update'], '_all_'=dict(fields=['script.repository'], pattern='github.com/user'))

        :return: The Tasks specified by the parameter combinations (see the parameters).
        """
        return cls.__get_tasks(task_ids=task_ids, project_name=project_name, tags=tags,
                               task_name=task_name, **(task_filter or {}))

    @classmethod
    def query_tasks(
            cls,
            project_name=None,  # type: Optional[Union[Sequence[str],str]]
            task_name=None,  # type: Optional[str]
            tags=None,  # type: Optional[Sequence[str]]
            additional_return_fields=None,  # type: Optional[Sequence[str]]
            task_filter=None,  # type: Optional[Dict]
    ):
        # type: (...) -> Union[List[str], List[Dict[str, str]]]
        """
        Get a list of Tasks ID matching the specific query/filter.
        Notice, if `additional_return_fields` is specified, returns a list of
        dictionaries with requested fields (dict per Task)

        :param str project_name: The project name of the Tasks to get. To get the experiment
            in all projects, use the default value of ``None``. (Optional)
            Use a list of strings for multiple optional project names.
        :param str task_name: The full name or partial name of the Tasks to match within the specified
            ``project_name`` (or all projects if ``project_name`` is ``None``).
            This method supports regular expressions for name matching. (Optional)
        :param str project_name: project name (str) the task belongs to (use None for all projects)
        :param str task_name: task name (str) in within the selected project
            Return any partial match of task_name, regular expressions matching is also supported
            If None is passed, returns all tasks within the project
        :param list tags: Filter based on the requested list of tags (strings) (Task must have all the listed tags)
            To exclude a tag add "-" prefix to the tag. Example: ["best", "-debug"]
        :param list additional_return_fields: Optional, if not provided return a list of Task IDs.
            If provided return dict per Task with the additional requested fields.
            Example: returned_fields=['last_updated', 'user', 'script.repository'] will return a list of dict:
                [{'id': 'task_id', 'last_update': datetime.datetime(),
                  'user': 'user_id', 'script.repository': 'https://github.com/user/'}, ]
        :param dict task_filter: filter and order Tasks. See service.tasks.GetAllRequest for details
            `parent`: (str) filter by parent task-id matching
            `search_text`: (str) free text search (in task fields comment/name/id)
            `status`: List[str] List of valid statuses
                (options are: "created", "queued", "in_progress", "stopped", "published", "publishing", "closed",
                "failed", "completed", "unknown")
            `type`: List[str] List of valid task type
                (options are: 'training', 'testing', 'inference', 'data_processing', 'application', 'monitor',
                'controller', 'optimizer', 'service', 'qc'. 'custom')
            `user`: List[str] Filter based on Task's user owner, provide list of valid user Ids.
            `order_by`: List[str] List of field names to order by. When search_text is used,
                Use '-' prefix to specify descending order. Optional, recommended when using page
                Example: order_by=['-last_update']
            `_all_`: dict(fields=[], pattern='')  Match string `pattern` (regular expression)
                appearing in All `fields`
                dict(fields=['script.repository'], pattern='github.com/user')
            `_any_`: dict(fields=[], pattern='')  Match string `pattern` (regular expression)
                appearing in Any of the `fields`
                dict(fields=['comment', 'name'], pattern='my comment')
            Examples:
                {'status': ['stopped'], 'order_by': ["-last_update"]}
                {'order_by'=['-last_update'], '_all_'=dict(fields=['script.repository'], pattern='github.com/user'))
        :return: The Tasks specified by the parameter combinations (see the parameters).
        """
        if tags:
            task_filter = task_filter or {}
            task_filter['tags'] = (task_filter.get('tags') or []) + list(tags)
        return_fields = {}
        if additional_return_fields:
            task_filter = task_filter or {}
            return_fields = set(list(additional_return_fields) + ['id'])
            task_filter['only_fields'] = (task_filter.get('only_fields') or []) + list(return_fields)

        results = cls._query_tasks(project_name=project_name, task_name=task_name, **(task_filter or {}))
        return [t.id for t in results] if not additional_return_fields else \
            [{k: cls._get_data_property(prop_path=k, data=r, raise_on_error=False, log_on_error=False)
              for k in return_fields}
             for r in results]

    @property
    def output_uri(self):
        # type: () -> str
        return self.storage_uri

    @property
    def last_worker(self):
        return self._data.last_worker

    @output_uri.setter
    def output_uri(self, value):
        # type: (Union[str, bool]) -> None

        # check if this is boolean
        if value is False:
            value = None
        elif value is True:
            value = self.__default_output_uri or self._get_default_report_storage_uri()

        # check if we have the correct packages / configuration
        if value and value != self.storage_uri:
            from .storage.helper import StorageHelper
            helper = StorageHelper.get(value)
            if not helper:
                raise ValueError("Could not get access credentials for '{}' "
                                 ", check configuration file ~/clearml.conf".format(value))
            helper.check_write_permissions(value)
        self.storage_uri = value

    @property
    def artifacts(self):
        # type: () -> Dict[str, Artifact]
        """
        A read-only dictionary of Task artifacts (name, artifact).

        :return: The artifacts.
        """
        if not Session.check_min_api_version('2.3'):
            return ReadOnlyDict()
        artifacts_pairs = []
        if self.data.execution and self.data.execution.artifacts:
            artifacts_pairs = [(a.key, Artifact(a)) for a in self.data.execution.artifacts]
        if self._artifacts_manager:
            artifacts_pairs += list(self._artifacts_manager.registered_artifacts.items())
        return ReadOnlyDict(artifacts_pairs)

    @property
    def models(self):
        # type: () -> Mapping[str, Sequence[Model]]
        """
        Read-only dictionary of the Task's loaded/stored models.

        :return: A dictionary-like object with "input"/"output" keys and input/output properties, pointing to a
            list-like object containing of Model objects. Each list-like object also acts as a dictionary, mapping
            model name to a appropriate model instance.

            Get input/output models:

            .. code-block:: py

                task.models.input
                task.models["input"]

                task.models.output
                task.models["output"]

            Get the last output model:

            .. code-block:: py

                task.models.output[-1]

            Get a model by name:

            .. code-block:: py

                task.models.output["model name"]
        """
        return self.get_models()

    @property
    def logger(self):
        # type: () -> Logger
        """
        Get a Logger object for reporting, for this task context. You can view all Logger report output associated with
        the Task for which this method is called, including metrics, plots, text, tables, and images, in the
        **ClearML Web-App (UI)**.

        :return: The Logger object for the current Task (experiment).
        """
        return self.get_logger()

    @classmethod
    def clone(
            cls,
            source_task=None,  # type: Optional[Union[Task, str]]
            name=None,  # type: Optional[str]
            comment=None,  # type: Optional[str]
            parent=None,  # type: Optional[str]
            project=None,  # type: Optional[str]
    ):
        # type: (...) -> Task
        """
        Create a duplicate (a clone) of a Task (experiment). The status of the cloned Task is ``Draft``
        and modifiable.

        Use this method to manage experiments and for autoML.

        :param str source_task: The Task to clone. Specify a Task object or a  Task ID. (Optional)
        :param str name: The name of the new cloned Task. (Optional)
        :param str comment: A comment / description for the new cloned Task. (Optional)
        :param str parent: The Id of the parent Task of the new Task.

            - If ``parent`` is not specified, then ``parent`` is set to ``source_task.parent``.
            - If ``parent`` is not specified and ``source_task.parent`` is not available, then
              ``parent`` set to ``source_task``.

        :param str project: The Id of the project in which to create the new Task.
            If ``None``, the new task inherits the original Task's project. (Optional)

        :return: The new cloned Task (experiment).
        """
        assert isinstance(source_task, (six.string_types, Task))
        if not Session.check_min_api_version('2.4'):
            raise ValueError("ClearML-server does not support DevOps features, "
                             "upgrade clearml-server to 0.12.0 or above")

        task_id = source_task if isinstance(source_task, six.string_types) else source_task.id
        if not parent:
            if isinstance(source_task, six.string_types):
                source_task = cls.get_task(task_id=source_task)
            parent = source_task.id if not source_task.parent else source_task.parent
        elif isinstance(parent, Task):
            parent = parent.id

        cloned_task_id = cls._clone_task(cloned_task_id=task_id, name=name, comment=comment,
                                         parent=parent, project=project)
        cloned_task = cls.get_task(task_id=cloned_task_id)
        return cloned_task

    @classmethod
    def enqueue(cls, task, queue_name=None, queue_id=None):
        # type: (Union[Task, str], Optional[str], Optional[str]) -> Any
        """
        Enqueue a Task for execution, by adding it to an execution queue.

        .. note::
           A worker daemon must be listening at the queue for the worker to fetch the Task and execute it,
           see `Use Case Examples <../clearml_agent_ref/#use-case-examples>`_ on the "ClearML Agent
           Reference page.

        :param Task/str task: The Task to enqueue. Specify a Task object or  Task ID.
        :param str queue_name: The name of the queue. If not specified, then ``queue_id`` must be specified.
        :param str queue_id: The Id of the queue. If not specified, then ``queue_name`` must be specified.

        :return: An enqueue JSON response.

            .. code-block:: javascript

               {
                    "queued": 1,
                    "updated": 1,
                    "fields": {
                        "status": "queued",
                        "status_reason": "",
                        "status_message": "",
                        "status_changed": "2020-02-24T15:05:35.426770+00:00",
                        "last_update": "2020-02-24T15:05:35.426770+00:00",
                        "execution.queue": "2bd96ab2d9e54b578cc2fb195e52c7cf"
                        }
                }

            - ``queued``  - The number of Tasks enqueued (an integer or ``null``).
            - ``updated`` - The number of Tasks updated (an integer or ``null``).
            - ``fields``

              - ``status`` - The status of the experiment.
              - ``status_reason`` - The reason for the last status change.
              - ``status_message`` - Information about the status.
              - ``status_changed`` - The last status change date and time (ISO 8601 format).
              - ``last_update`` - The last Task update time, including Task creation, update, change, or events for
                this task (ISO 8601 format).
              - ``execution.queue`` - The Id of the queue where the Task is enqueued. ``null`` indicates not enqueued.

        """
        assert isinstance(task, (six.string_types, Task))
        if not Session.check_min_api_version('2.4'):
            raise ValueError("ClearML-server does not support DevOps features, "
                             "upgrade clearml-server to 0.12.0 or above")

        # make sure we have wither name ot id
        mutually_exclusive(queue_name=queue_name, queue_id=queue_id)

        task_id = task if isinstance(task, six.string_types) else task.id
        session = cls._get_default_session()
        if not queue_id:
            queue_id = get_queue_id(session, queue_name)
            if not queue_id:
                raise ValueError('Could not find queue named "{}"'.format(queue_name))

        req = tasks.EnqueueRequest(task=task_id, queue=queue_id)
        res = cls._send(session=session, req=req)
        if not res.ok():
            raise ValueError(res.response)
        resp = res.response
        return resp

    @classmethod
    def get_num_enqueued_tasks(cls, queue_name=None, queue_id=None):
        # type: (Optional[str], Optional[str]) -> int
        """
        Get the number of tasks enqueued in a given queue.

        :param queue_name: The name of the queue. If not specified, then ``queue_id`` must be specified
        :param queue_id: The id of the queue. If not specified, then ``queue_name`` must be specified

        :return: The number of tasks enqueued in the given queue
        """
        if not Session.check_min_api_server_version("2.20"):
            raise ValueError("You version of clearml-server does not support the 'queues.get_num_entries' endpoint")
        mutually_exclusive(queue_name=queue_name, queue_id=queue_id)
        session = cls._get_default_session()
        if not queue_id:
            queue_id = get_queue_id(session, queue_name)
            if not queue_id:
                raise ValueError('Could not find queue named "{}"'.format(queue_name))
        result = get_num_enqueued_tasks(session, queue_id)
        if result is None:
            raise ValueError("Could not query the number of enqueued tasks in queue with ID {}".format(queue_id))
        return result

    @classmethod
    def dequeue(cls, task):
        # type: (Union[Task, str]) -> Any
        """
        Dequeue (remove) a Task from an execution queue.

        :param Task/str task: The Task to dequeue. Specify a Task object or  Task ID.

        :return: A dequeue JSON response.

        .. code-block:: javascript

           {
                "dequeued": 1,
                "updated": 1,
                "fields": {
                    "status": "created",
                    "status_reason": "",
                    "status_message": "",
                    "status_changed": "2020-02-24T16:43:43.057320+00:00",
                    "last_update": "2020-02-24T16:43:43.057320+00:00",
                    "execution.queue": null
                    }
            }

        - ``dequeued``  - The number of Tasks enqueued (an integer or ``null``).
        - ``fields``

          - ``status`` - The status of the experiment.
          - ``status_reason`` - The reason for the last status change.
          - ``status_message`` - Information about the status.
          - ``status_changed`` - The last status change date and time in ISO 8601 format.
          - ``last_update`` - The last time the Task was created, updated,
                changed or events for this task were reported.
          - ``execution.queue`` - The Id of the queue where the Task is enqueued. ``null`` indicates not enqueued.

        - ``updated`` - The number of Tasks updated (an integer or ``null``).

        """
        assert isinstance(task, (six.string_types, Task))
        if not Session.check_min_api_version('2.4'):
            raise ValueError("ClearML-server does not support DevOps features, "
                             "upgrade clearml-server to 0.12.0 or above")

        task_id = task if isinstance(task, six.string_types) else task.id
        session = cls._get_default_session()
        req = tasks.DequeueRequest(task=task_id)
        res = cls._send(session=session, req=req)
        resp = res.response
        return resp

    def set_progress(self, progress):
        # type: (int) -> ()
        """
        Sets Task's progress (0 - 100)
        Progress is a field computed and reported by the user.

        :param progress: numeric value (0 - 100)
        """
        if not isinstance(progress, int) or progress < 0 or progress > 100:
            self.log.warning("Can't set progress {} as it is not and int between 0 and 100".format(progress))
            return
        self._set_runtime_properties({"progress": str(progress)})

    def get_progress(self):
        # type: () -> (Optional[int])
        """
        Gets Task's progress (0 - 100)

        :return: Task's progress as an int.
            In case the progress doesn't exist, None will be returned
        """
        progress = self._get_runtime_properties().get("progress")
        if progress is None or not progress.isnumeric():
            return None
        return int(progress)

    def add_tags(self, tags):
        # type: (Union[Sequence[str], str]) -> None
        """
        Add Tags to this task. Old tags are not deleted. When executing a Task (experiment) remotely,
        this method has no effect).

        :param tags: A list of tags which describe the Task to add.
        """

        if isinstance(tags, six.string_types):
            tags = tags.split(" ")

        self.data.tags = list(set((self.data.tags or []) + tags))
        self._edit(tags=self.data.tags)

    def connect(self, mutable, name=None):
        # type: (Any, Optional[str]) -> Any
        """
        Connect an object to a Task object. This connects an experiment component (part of an experiment) to the
        experiment. For example, connect hyperparameters or models.

        :param object mutable: The experiment component to connect. The object can be any object Task supports
            integrating, including:

            - argparse - An argparse object for parameters.
            - dict - A dictionary for parameters.
            - TaskParameters - A TaskParameters object.
            - Model - A model object for initial model warmup, or for model update/snapshot uploading.
            - Class type - A Class type, storing all class properties (excluding '_' prefix properties)
            - Object - A class instance, storing all instance properties (excluding '_' prefix properties)

        :param str name: A section name associated with the connected object, if 'name' is None defaults to 'General'
            Currently only supported for `dict` / `TaskParameter` objects
            Examples:
            name='General' will put the connected dictionary under the General section in the hyper-parameters
            name='Train' will put the connected dictionary under the Train section in the hyper-parameters

        :return: The result returned when connecting the object, if supported.

        :raise: Raise an exception on unsupported objects.
        """
        # dispatching by match order
        dispatch = (
            (OutputModel, self._connect_output_model),
            (InputModel, self._connect_input_model),
            (ArgumentParser, self._connect_argparse),
            (dict, self._connect_dictionary),
            (TaskParameters, self._connect_task_parameters),
            (type, self._connect_object),
            (object, self._connect_object),
        )

        multi_config_support = Session.check_min_api_version('2.9')
        if multi_config_support and not name and not isinstance(mutable, (OutputModel, InputModel)):
            name = self._default_configuration_section_name

        if not multi_config_support and name and name != self._default_configuration_section_name:
            raise ValueError("Multiple configurations is not supported with the current 'clearml-server', "
                             "please upgrade to the latest version")

        for mutable_type, method in dispatch:
            if isinstance(mutable, mutable_type):
                return method(mutable, name=name)

        raise Exception('Unsupported mutable type %s: no connect function found' % type(mutable).__name__)

    def connect_configuration(self, configuration, name=None, description=None):
        # type: (Union[Mapping, list, Path, str], Optional[str], Optional[str]) -> Union[dict, Path, str]
        """
        Connect a configuration dictionary or configuration file (pathlib.Path / str) to a Task object.
        This method should be called before reading the configuration file.

        Later, when creating an output model, the model will include the contents of the configuration dictionary
        or file.

        For example, a local file:

        .. code-block:: py

           config_file = task.connect_configuration(config_file)
           my_params = json.load(open(config_file,'rt'))

        A parameter dictionary/list:

        .. code-block:: py

           my_params = task.connect_configuration(my_params)

        :param configuration: The configuration. This is usually the configuration used in the model training process.
            Specify one of the following:

            - A dictionary/list - A dictionary containing the configuration. ClearML stores the configuration in
              the **ClearML Server** (backend), in a HOCON format (JSON-like format) which is editable.
            - A ``pathlib2.Path`` string - A path to the configuration file. ClearML stores the content of the file.
              A local path must be relative path. When executing a Task remotely in a worker, the contents brought
              from the **ClearML Server** (backend) overwrites the contents of the file.

        :param str name: Configuration section name. default: 'General'
            Allowing users to store multiple configuration dicts/files

        :param str description: Configuration section description (text). default: None

        :return: If a dictionary is specified, then a dictionary is returned. If pathlib2.Path / string is
            specified, then a path to a local configuration file is returned. Configuration object.
        """
        pathlib_Path = None  # noqa
        if not isinstance(configuration, (dict, list, Path, six.string_types)):
            try:
                from pathlib import Path as pathlib_Path  # noqa
            except ImportError:
                pass
            if not pathlib_Path or not isinstance(configuration, pathlib_Path):
                raise ValueError("connect_configuration supports `dict`, `str` and 'Path' types, "
                                 "{} is not supported".format(type(configuration)))

        multi_config_support = Session.check_min_api_version('2.9')
        if multi_config_support and not name:
            name = self._default_configuration_section_name

        if not multi_config_support and name and name != self._default_configuration_section_name:
            raise ValueError("Multiple configurations is not supported with the current 'clearml-server', "
                             "please upgrade to the latest version")

        # parameter dictionary
        if isinstance(configuration, (dict, list,)):
            def _update_config_dict(task, config_dict):
                if multi_config_support:
                    # noinspection PyProtectedMember
                    task._set_configuration(
                        name=name, description=description, config_type='dictionary', config_dict=config_dict)
                else:
                    # noinspection PyProtectedMember
                    task._set_model_config(config_dict=config_dict)

            def get_dev_config(configuration_):
                if multi_config_support:
                    self._set_configuration(
                        name=name, description=description, config_type="dictionary", config_dict=configuration_
                    )
                else:
                    self._set_model_config(config_dict=configuration)
                if isinstance(configuration_, dict):
                    configuration_ = ProxyDictPostWrite(self, _update_config_dict, **configuration_)
                return configuration_

            if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()):
                configuration = get_dev_config(configuration)
            else:
                # noinspection PyBroadException
                try:
                    remote_configuration = self._get_configuration_dict(name=name) \
                        if multi_config_support else self._get_model_config_dict()
                except Exception:
                    remote_configuration = None

                if remote_configuration is None:
                    LoggerRoot.get_base_logger().warning(
                        "Could not retrieve remote configuration named \'{}\'\n"
                        "Using default configuration: {}".format(name, str(configuration)))
                    # update back configuration section
                    if multi_config_support:
                        self._set_configuration(
                            name=name, description=description,
                            config_type='dictionary', config_dict=configuration)
                    return configuration

                if not remote_configuration:
                    configuration = get_dev_config(configuration)
                elif isinstance(configuration, dict):
                    configuration.clear()
                    configuration.update(remote_configuration)
                    configuration = ProxyDictPreWrite(False, False, **configuration)
                elif isinstance(configuration, list):
                    configuration.clear()
                    configuration.extend(remote_configuration)

            return configuration

        # it is a path to a local file
        if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()):
            # check if not absolute path
            configuration_path = Path(configuration)
            if not configuration_path.is_file():
                ValueError("Configuration file does not exist")
            try:
                with open(configuration_path.as_posix(), 'rt') as f:
                    configuration_text = f.read()
            except Exception:
                raise ValueError("Could not connect configuration file {}, file could not be read".format(
                    configuration_path.as_posix()))
            if multi_config_support:
                self._set_configuration(
                    name=name, description=description,
                    config_type=configuration_path.suffixes[-1].lstrip('.')
                    if configuration_path.suffixes and configuration_path.suffixes[-1] else 'file',
                    config_text=configuration_text)
            else:
                self._set_model_config(config_text=configuration_text)
            return configuration
        else:
            configuration_text = self._get_configuration_text(name=name) if multi_config_support \
                else self._get_model_config_text()
            if configuration_text is None:
                LoggerRoot.get_base_logger().warning(
                    "Could not retrieve remote configuration named \'{}\'\n"
                    "Using default configuration: {}".format(name, str(configuration)))
                # update back configuration section
                if multi_config_support:
                    configuration_path = Path(configuration)
                    if configuration_path.is_file():
                        with open(configuration_path.as_posix(), 'rt') as f:
                            configuration_text = f.read()

                        self._set_configuration(
                            name=name, description=description,
                            config_type=configuration_path.suffixes[-1].lstrip('.')
                            if configuration_path.suffixes and configuration_path.suffixes[-1] else 'file',
                            config_text=configuration_text)
                return configuration

            configuration_path = Path(configuration)
            fd, local_filename = mkstemp(prefix='clearml_task_config_',
                                         suffix=configuration_path.suffixes[-1] if
                                         configuration_path.suffixes else '.txt')
            os.write(fd, configuration_text.encode('utf-8'))
            os.close(fd)
            if pathlib_Path:
                return pathlib_Path(local_filename)
            return Path(local_filename) if isinstance(configuration, Path) else local_filename

    def connect_label_enumeration(self, enumeration):
        # type: (Dict[str, int]) -> Dict[str, int]
        """
        Connect a label enumeration dictionary to a Task (experiment) object.

        Later, when creating an output model, the model will include the label enumeration dictionary.

        :param dict enumeration: A label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }

        :return: The label enumeration dictionary (JSON).
        """
        if not isinstance(enumeration, dict):
            raise ValueError("connect_label_enumeration supports only `dict` type, "
                             "{} is not supported".format(type(enumeration)))

        if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()):
            self.set_model_label_enumeration(enumeration)
        else:
            # pop everything
            enumeration.clear()
            enumeration.update(self.get_labels_enumeration())
        return enumeration

    def get_logger(self):
        # type: () -> Logger
        """
        Get a Logger object for reporting, for this task context. You can view all Logger report output associated with
        the Task for which this method is called, including metrics, plots, text, tables, and images, in the
        **ClearML Web-App (UI)**.

        :return: The Logger for the Task (experiment).
        """
        return self._get_logger(auto_connect_streams=self._log_to_backend)

    def mark_started(self, force=False):
        # type: (bool) -> ()
        """
        Manually mark a Task as started (happens automatically)

        :param bool force: If True the task status will be changed to `started` regardless of the current Task state.
        """
        # UI won't let us see metrics if we're not started
        self.started(force=force)
        self.reload()

    def mark_stopped(self, force=False, status_message=None):
        # type: (bool, Optional[str]) -> ()
        """
        Manually mark a Task as stopped (also used in :meth:`_at_exit`)

        :param bool force: If True the task status will be changed to `stopped` regardless of the current Task state.
        :param str status_message: Optional, add status change message to the stop request.
            This message will be stored as status_message on the Task's info panel
        """
        # flush any outstanding logs
        self.flush(wait_for_uploads=True)
        # mark task as stopped
        self.stopped(force=force, status_message=str(status_message) if status_message else None)

    def flush(self, wait_for_uploads=False):
        # type: (bool) -> bool
        """
        Flush any outstanding reports or console logs.

        :param bool wait_for_uploads: Wait for all outstanding uploads to complete

            - ``True`` - Wait
            - ``False`` - Do not wait (default)
        """

        # make sure model upload is done
        if BackendModel.get_num_results() > 0 and wait_for_uploads:
            BackendModel.wait_for_results()

        # flush any outstanding logs
        if self._logger:
            # noinspection PyProtectedMember
            self._logger._flush_stdout_handler()
        if self.__reporter:
            self.__reporter.flush()
            if wait_for_uploads:
                self.__reporter.wait_for_events()

        LoggerRoot.flush()

        return True

    def reset(self, set_started_on_success=False, force=False):
        # type: (bool, bool) -> None
        """
        Reset a Task. ClearML reloads a Task after a successful reset.
        When a worker executes a Task remotely, the Task does not reset unless
        the ``force`` parameter is set to ``True`` (this avoids accidentally clearing logs and metrics).

        :param bool set_started_on_success: If successful, automatically set the Task to `started`

            - ``True`` - If successful, set to started.
            - ``False`` - If successful, do not set to started. (default)

        :param bool force: Force a Task reset, even when executing the Task (experiment) remotely in a worker

            - ``True`` - Force
            - ``False`` - Do not force (default)
        """
        if not running_remotely() or not self.is_main_task() or force:
            super(Task, self).reset(set_started_on_success=set_started_on_success, force=force)

    def close(self):
        """
        Close the current Task. Enables you to manually shutdown the task.

        .. warning::
           Only call :meth:`Task.close` if you are certain the Task is not needed.
        """
        if self._at_exit_called:
            return

        # store is main before we call at_exit, because will will Null it
        is_main = self.is_main_task()
        is_sub_process = self.__is_subprocess()

        # wait for repository detection (5 minutes should be reasonable time to detect all packages)
        if self._logger and not self.__is_subprocess():
            self._wait_for_repo_detection(timeout=300.)

        self.__shutdown()
        # unregister atexit callbacks and signal hooks, if we are the main task
        if is_main:
            self.__register_at_exit(None)
            if not is_sub_process:
                # make sure we enable multiple Task.init callas with reporting sub-processes
                BackgroundMonitor.clear_main_process(self)
                # noinspection PyProtectedMember
                Logger._remove_std_logger()

                # unbind everything
                PatchHydra.update_current_task(None)
                PatchedJoblib.update_current_task(None)
                PatchedMatplotlib.update_current_task(None)
                PatchAbsl.update_current_task(None)
                TensorflowBinding.update_current_task(None)
                PatchPyTorchModelIO.update_current_task(None)
                PatchMegEngineModelIO.update_current_task(None)
                PatchXGBoostModelIO.update_current_task(None)
                PatchCatBoostModelIO.update_current_task(None)
                PatchFastai.update_current_task(None)
                PatchLIGHTgbmModelIO.update_current_task(None)
                EnvironmentBind.update_current_task(None)
                PatchJsonArgParse.update_current_task(None)
                PatchOsFork.patch_fork(None)

    def delete(
            self,
            delete_artifacts_and_models=True,
            skip_models_used_by_other_tasks=True,
            raise_on_error=False,
            callback=None,
    ):
        # type: (bool, bool, bool, Callable[[str, str], bool]) -> bool
        """
        Delete the task as well as its output models and artifacts.
        Models and artifacts are deleted from their storage locations, each using its URI.

        Note: in order to delete models and artifacts using their URI, make sure the proper storage credentials are
        configured in your configuration file (e.g. if an artifact is stored in S3, make sure sdk.aws.s3.credentials
        are properly configured and that you have delete permission in the related buckets).

        :param delete_artifacts_and_models: If True, artifacts and models would also be deleted (default True).
                                            If callback is provided, this argument is ignored.
        :param skip_models_used_by_other_tasks: If True, models used by other tasks would not be deleted (default True)
        :param raise_on_error: If True an exception will be raised when encountering an error.
                               If False an error would be printed and no exception will be raised.
        :param callback: An optional callback accepting a uri type (string) and a uri (string) that will be called
                         for each artifact and model. If provided, the delete_artifacts_and_models is ignored.
                         Return True to indicate the artifact/model should be deleted or False otherwise.
        :return: True if the task was deleted successfully.
        """
        if not running_remotely() or not self.is_main_task():
            return super(Task, self)._delete(
                delete_artifacts_and_models=delete_artifacts_and_models,
                skip_models_used_by_other_tasks=skip_models_used_by_other_tasks,
                raise_on_error=raise_on_error,
                callback=callback,
            )
        return False

    def register_artifact(self, name, artifact, metadata=None, uniqueness_columns=True):
        # type: (str, pandas.DataFrame, Dict, Union[bool, Sequence[str]]) -> None
        """
        Register (add) an artifact for the current Task. Registered artifacts are dynamically synchronized with the
        **ClearML Server** (backend). If a registered artifact is updated, the update is stored in the
        **ClearML Server** (backend). Registered artifacts are primarily used for Data Auditing.

        The currently supported registered artifact object type is a pandas.DataFrame.

        See also :meth:`Task.unregister_artifact` and :meth:`Task.get_registered_artifacts`.

        .. note::
           ClearML also supports uploaded artifacts which are one-time uploads of static artifacts that are not
           dynamically synchronized with the **ClearML Server** (backend). These static artifacts include
           additional object types. For more information, see :meth:`Task.upload_artifact`.

        :param str name: The name of the artifact.

         .. warning::
            If an artifact with the same name was previously registered, it is overwritten.
        :param object artifact: The artifact object.
        :param dict metadata: A dictionary of key-value pairs for any metadata. This dictionary appears with the
            experiment in the **ClearML Web-App (UI)**, **ARTIFACTS** tab.
        :param uniqueness_columns: A Sequence of columns for artifact uniqueness comparison criteria, or the default
            value of ``True``. If ``True``, the artifact uniqueness comparison criteria is all the columns,
            which is the same as ``artifact.columns``.
        """
        if not isinstance(uniqueness_columns, CollectionsSequence) and uniqueness_columns is not True:
            raise ValueError('uniqueness_columns should be a List (sequence) or True')
        if isinstance(uniqueness_columns, str):
            uniqueness_columns = [uniqueness_columns]
        self._artifacts_manager.register_artifact(
            name=name, artifact=artifact, metadata=metadata, uniqueness_columns=uniqueness_columns)

    def unregister_artifact(self, name):
        # type: (str) -> None
        """
        Unregister (remove) a registered artifact. This removes the artifact from the watch list that ClearML uses
        to synchronize artifacts with the **ClearML Server** (backend).

        .. important::
           - Calling this method does not remove the artifact from a Task. It only stops ClearML from
             monitoring the artifact.
           - When this method is called, ClearML immediately takes the last snapshot of the artifact.
        """
        self._artifacts_manager.unregister_artifact(name=name)

    def get_registered_artifacts(self):
        # type: () -> Dict[str, Artifact]
        """
        Get a dictionary containing the Task's registered (dynamically synchronized) artifacts (name, artifact object).

        .. note::
           After calling ``get_registered_artifacts``, you can still modify the registered artifacts.

        :return: The registered (dynamically synchronized) artifacts.
        """
        return self._artifacts_manager.registered_artifacts

    def upload_artifact(
            self,
            name,  # type: str
            artifact_object,  # type: Union[str, Mapping, pandas.DataFrame, numpy.ndarray, Image.Image, Any]
            metadata=None,  # type: Optional[Mapping]
            delete_after_upload=False,  # type: bool
            auto_pickle=True,  # type: bool
            preview=None,  # type: Any
            wait_on_upload=False,  # type: bool
            extension_name=None,  # type: Optional[str]
    ):
        # type: (...) -> bool
        """
        Upload (add) a static artifact to a Task object. The artifact is uploaded in the background.

        The currently supported upload (static) artifact types include:

        - string / pathlib2.Path - A path to artifact file. If a wildcard or a folder is specified, then ClearML
          creates and uploads a ZIP file.
        - dict - ClearML stores a dictionary as ``.json`` (or see ``extension_name``) file and uploads it.
        - pandas.DataFrame - ClearML stores a pandas.DataFrame as ``.csv.gz`` (compressed CSV)
            (or see ``extension_name``) file and uploads it.
        - numpy.ndarray - ClearML stores a numpy.ndarray as ``.npz`` (or see ``extension_name``)
            file and uploads it.
        - PIL.Image - ClearML stores a PIL.Image as ``.png`` (or see ``extension_name``) file and uploads it.
        - Any - If called with auto_pickle=True, the object will be pickled and uploaded.

        :param str name: The artifact name.

            .. warning::
               If an artifact with the same name was previously uploaded, then it is overwritten.

        :param object artifact_object:  The artifact object.
        :param dict metadata: A dictionary of key-value pairs for any metadata. This dictionary appears with the
            experiment in the **ClearML Web-App (UI)**, **ARTIFACTS** tab.
        :param bool delete_after_upload: After the upload, delete the local copy of the artifact

            - ``True`` - Delete the local copy of the artifact.
            - ``False`` - Do not delete. (default)

        :param bool auto_pickle: If True (default) and the artifact_object is not one of the following types:
            pathlib2.Path, dict, pandas.DataFrame, numpy.ndarray, PIL.Image, url (string), local_file (string)
            the artifact_object will be pickled and uploaded as pickle file artifact (with file extension .pkl)

        :param Any preview: The artifact preview

        :param bool wait_on_upload: Whether the upload should be synchronous, forcing the upload to complete
            before continuing.

        :param str extension_name: File extension which indicates the format the artifact should be stored as.
            The following are supported, depending on the artifact type
            (default value applies when extension_name is None):
        - dict - ``.json``, ``.yaml`` (default ``.json``)
        - pandas.DataFrame - ``.csv.gz``, ``.parquet``, ``.feather``, ``.pickle`` (default ``.csv.gz``)
        - numpy.ndarray - ``.npz``, ``.csv.gz`` (default ``.npz``)
        - PIL.Image - whatever extensions PIL supports (default ``.png``)

        :return: The status of the upload.

        - ``True`` - Upload succeeded.
        - ``False`` - Upload failed.

        :raise: If the artifact object type is not supported, raise a ``ValueError``.
        """
        return self._artifacts_manager.upload_artifact(
            name=name, artifact_object=artifact_object, metadata=metadata, delete_after_upload=delete_after_upload,
            auto_pickle=auto_pickle, preview=preview, wait_on_upload=wait_on_upload, extension_name=extension_name)

    def get_models(self):
        # type: () -> Mapping[str, Sequence[Model]]
        """
        Return a dictionary with {'input': [], 'output': []} loaded/stored models of the current Task
        Input models are files loaded in the task, either manually or automatically logged
        Output models are files stored in the task, either manually or automatically logged
        Automatically logged frameworks are for example: TensorFlow, Keras, PyTorch, ScikitLearn(joblib) etc.

        :return: A dictionary-like object with "input"/"output" keys and input/output properties, pointing to a
            list-like object containing of Model objects. Each list-like object also acts as a dictionary, mapping
            model name to a appropriate model instance.

            Example:

            .. code-block:: py

                {'input': [clearml.Model()], 'output': [clearml.Model()]}

        """
        return TaskModels(self)

    def is_current_task(self):
        # type: () -> bool
        """
        .. deprecated:: 0.13.0
           This method is deprecated. Use :meth:`Task.is_main_task` instead.

        Is this Task object the main execution Task (initially returned by :meth:`Task.init`)

        :return: Is this Task object the main execution Task

            - ``True`` - Is the main execution Task.
            - ``False`` - Is not the main execution Task.

        """
        return self.is_main_task()

    def is_main_task(self):
        # type: () -> bool
        """
        Is this Task object the main execution Task (initially returned by :meth:`Task.init`)

        .. note::
           If :meth:`Task.init` was never called, this method will *not* create
           it, making this test more efficient than:

           .. code-block:: py

              Task.init() == task

        :return: Is this Task object the main execution Task

            - ``True`` - Is the main execution Task.
            - ``False`` - Is not the main execution Task.

        """
        return self is self.__main_task

    def set_model_config(self, config_text=None, config_dict=None):
        # type: (Optional[str], Optional[Mapping]) -> None
        """
        .. deprecated:: 0.14.1
            Use :meth:`Task.connect_configuration` instead.
        """
        self._set_model_config(config_text=config_text, config_dict=config_dict)

    def get_model_config_text(self):
        # type: () -> str
        """
        .. deprecated:: 0.14.1
            Use :meth:`Task.connect_configuration` instead.
        """
        return self._get_model_config_text()

    def get_model_config_dict(self):
        # type: () -> Dict
        """
        .. deprecated:: 0.14.1
            Use :meth:`Task.connect_configuration` instead.
        """
        return self._get_model_config_dict()

    def set_model_label_enumeration(self, enumeration=None):
        # type: (Optional[Mapping[str, int]]) -> ()
        """
        Set the label enumeration for the Task object before creating an output model.
        Later, when creating an output model, the model will inherit these properties.

        :param dict enumeration: A label enumeration dictionary of string (label) to integer (value) pairs.

            For example:

            .. code-block:: javascript

               {
                    'background': 0,
                    'person': 1
               }
        """
        super(Task, self).set_model_label_enumeration(enumeration=enumeration)

    def get_last_iteration(self):
        # type: () -> int
        """
        Get the last reported iteration, which is the last iteration for which the Task reported a metric.

        .. note::
           The maximum reported iteration is not in the local cache. This method
           sends a request to the **ClearML Server** (backend).

        :return: The last reported iteration number.
        """
        self._reload_last_iteration()
        return max(self.data.last_iteration or 0, self.__reporter.max_iteration if self.__reporter else 0)

    def set_initial_iteration(self, offset=0):
        # type: (int) -> int
        """
        Set initial iteration, instead of zero. Useful when continuing training from previous checkpoints

        :param int offset: Initial iteration (at starting point)
        :return: Newly set initial offset.
        """
        return super(Task, self).set_initial_iteration(offset=offset)

    def get_initial_iteration(self):
        # type: () -> int
        """
        Return the initial iteration offset, default is 0
        Useful when continuing training from previous checkpoints

        :return: Initial iteration offset.
        """
        return super(Task, self).get_initial_iteration()

    def get_last_scalar_metrics(self):
        # type: () -> Dict[str, Dict[str, Dict[str, float]]]
        """
        Get the last scalar metrics which the Task reported. This is a nested dictionary, ordered by title and series.

        For example:

        .. code-block:: javascript

           {
            'title': {
                'series': {
                    'last': 0.5,
                    'min': 0.1,
                    'max': 0.9
                    }
                }
            }

        :return: The last scalar metrics.
        """
        self.reload()
        metrics = self.data.last_metrics
        scalar_metrics = dict()
        for i in metrics.values():
            for j in i.values():
                scalar_metrics.setdefault(j['metric'], {}).setdefault(
                    j['variant'], {'last': j['value'], 'min': j['min_value'], 'max': j['max_value']})
        return scalar_metrics

    def get_parameters_as_dict(self, cast=False):
        # type: (bool) -> Dict
        """
        Get the Task parameters as a raw nested dictionary.

        .. note::
           If `cast` is False (default) The values are not parsed. They are returned as is.

        :param cast: If True, cast the parameter to the original type. Default False,
            values are returned in their string representation

        """
        return naive_nested_from_flat_dictionary(self.get_parameters(cast=cast))

    def set_parameters_as_dict(self, dictionary):
        # type: (Dict) -> None
        """
        Set the parameters for the Task object from a dictionary. The dictionary can be nested.
        This does not link the dictionary to the Task object. It does a one-time update. This
        is the same behavior as the :meth:`Task.connect` method.
        """
        self._arguments.copy_from_dict(flatten_dictionary(dictionary))

    def get_user_properties(self, value_only=False):
        # type: (bool) -> Dict[str, Union[str, dict]]
        """
        Get user properties for this task.
        Returns a dictionary mapping user property name to user property details dict.
        :param value_only: If True, returned user property details will be a string representing the property value.
        """
        if not Session.check_min_api_version("2.9"):
            self.log.info("User properties are not supported by the server")
            return {}

        section = "properties"

        params = self._hyper_params_manager.get_hyper_params(
            sections=[section], projector=attrgetter("value") if value_only else None
        )

        return dict(params.get(section, {}))

    def set_user_properties(
            self,
            *iterables,  # type: Union[Mapping[str, Union[str, dict, None]], Iterable[dict]]
            **properties  # type: Union[str, dict, int, float, None]
    ):
        # type: (...) -> bool
        """
        Set user properties for this task.
        A user property can contain the following fields (all of type string):
        name / value / description / type

        Examples:
            task.set_user_properties(backbone='great', stable=True)
            task.set_user_properties(backbone={"type": int, "description": "network type", "value": "great"}, )
            task.set_user_properties(
                {"name": "backbone", "description": "network type", "value": "great"},
                {"name": "stable", "description": "is stable", "value": True},
            )

        :param iterables: Properties iterables, each can be:
            * A dictionary of string key (name) to either a string value (value) a dict (property details). If the value
                is a dict, it must contain a "value" field. For example:

                .. code-block:: javascript

                    {
                        "property_name": {"description": "This is a user property", "value": "property value"},
                        "another_property_name": {"description": "This is user property", "value": "another value"},
                        "yet_another_property_name": "some value"
                    }

            * An iterable of dicts (each representing property details). Each dict must contain a "name" field and a
                "value" field. For example:

                .. code-block:: javascript

                    [
                        {
                            "name": "property_name",
                            "description": "This is a user property",
                            "value": "property value"
                        },
                        {
                            "name": "another_property_name",
                            "description": "This is another user property",
                            "value": "another value"
                        }
                    ]

        :param properties: Additional properties keyword arguments. Key is the property name, and value can be
            a string (property value) or a dict (property details). If the value is a dict, it must contain a "value"
            field. For example:

            .. code-block:: javascript

            {
                "property_name": "string as property value",
                "another_property_name": {
                    "type": "string",
                    "description": "This is user property",
                    "value": "another value"
                }
            }

        """
        if not Session.check_min_api_version("2.9"):
            self.log.info("User properties are not supported by the server")
            return False

        return self._hyper_params_manager.edit_hyper_params(
            iterables=list(properties.items()) + (
                list(iterables.items()) if isinstance(iterables, dict) else list(iterables)),
            replace='none',
            force_section="properties",
        )

    def get_script(self):
        # type: (...) -> Mapping[str, Optional[str]]
        """
        Get task's script details.

        Returns a dictionary containing the script details.
        :return: Dictionary with script properties e.g.
                {
                'working_dir': 'examples/reporting',
                'entry_point': 'artifacts.py',
                'branch': 'master',
                'repository': 'https://github.com/allegroai/clearml.git'
                }
        """
        script = self.data.script
        return {
            "working_dir": script.working_dir,
            "entry_point": script.entry_point,
            "branch": script.branch,
            "repository": script.repository
        }

    def set_script(
            self,
            repository=None,  # type: Optional[str]
            branch=None,  # type: Optional[str]
            commit=None,  # type: Optional[str]
            diff=None,  # type: Optional[str]
            working_dir=None,  # type: Optional[str]
            entry_point=None,  # type: Optional[str]
    ):
        # type: (...) -> None
        """
        Set task's script.

        Examples:
            task.set_script(repository='https://github.com/allegroai/clearml.git,
                            branch='main',
                            working_dir='examples/reporting',
                            entry_point='artifacts.py')

        :param repository: Optional, URL of remote repository. use empty string ("") to clear repository entry.
        :param branch: Optional, Select specific repository branch / tag. use empty string ("") to clear branch entry.
        :param commit: Optional, set specific git commit id. use empty string ("") to clear commit id entry.
        :param diff: Optional, set "git diff" section. use empty string ("") to clear git-diff entry.
        :param working_dir: Optional, Working directory to launch the script from.
        :param entry_point: Optional, Path to execute within the repository.

        """
        self.reload()
        script = self.data.script
        if repository is not None:
            script.repository = str(repository) or None
        if branch is not None:
            script.branch = str(branch) or None
            if script.tag:
                script.tag = None
        if commit is not None:
            script.version_num = str(commit) or None
        if diff is not None:
            script.diff = str(diff) or None
        if working_dir is not None:
            script.working_dir = str(working_dir)
        if entry_point is not None:
            script.entry_point = str(entry_point)
        # noinspection PyProtectedMember
        self._update_script(script=script)

    def delete_user_properties(self, *iterables):
        # type: (Iterable[Union[dict, Iterable[str, str]]]) -> bool
        """
        Delete hyper-parameters for this task.
        :param iterables: Hyper parameter key iterables. Each an iterable whose possible values each represent
        a hyper-parameter entry to delete, value formats are:

            * A dictionary containing a 'section' and 'name' fields
            * An iterable (e.g. tuple, list etc.) whose first two items denote 'section' and 'name'
        """
        if not Session.check_min_api_version("2.9"):
            self.log.info("User properties are not supported by the server")
            return False

        return self._hyper_params_manager.delete_hyper_params(*iterables)

    def set_base_docker(
            self,
            docker_cmd=None,  # type: Optional[str]
            docker_image=None,  # type: Optional[str]
            docker_arguments=None,  # type: Optional[Union[str, Sequence[str]]]
            docker_setup_bash_script=None  # type: Optional[Union[str, Sequence[str]]]
    ):
        # type: (...) -> ()
        """
        Set the base docker image for this experiment
        If provided, this value will be used by clearml-agent to execute this experiment
        inside the provided docker image.
        When running remotely the call is ignored

        :param docker_cmd: Deprecated! compound docker container image + arguments
            (example: 'nvidia/cuda:11.1 -e test=1') Deprecated, use specific arguments.
        :param docker_image: docker container image (example: 'nvidia/cuda:11.1')
        :param docker_arguments: docker execution parameters (example: '-e ENV=1')
        :param docker_setup_bash_script: bash script to run at the
            beginning of the docker before launching the Task itself. example: ['apt update', 'apt-get install -y gcc']
        """
        if not self.running_locally() and self.is_main_task():
            return

        super(Task, self).set_base_docker(
            docker_cmd=docker_cmd or docker_image,
            docker_arguments=docker_arguments,
            docker_setup_bash_script=docker_setup_bash_script
        )

    def set_resource_monitor_iteration_timeout(self, seconds_from_start=1800):
        # type: (float) -> bool
        """
        Set the ResourceMonitor maximum duration (in seconds) to wait until first scalar/plot is reported.
        If timeout is reached without any reporting, the ResourceMonitor will start reporting machine statistics based
        on seconds from Task start time (instead of based on iteration)

        :param seconds_from_start: Maximum number of seconds to wait for scalar/plot reporting before defaulting
            to machine statistics reporting based on seconds from experiment start time
        :return: True if success
        """
        if not self._resource_monitor:
            return False
        self._resource_monitor.wait_for_first_iteration = seconds_from_start
        self._resource_monitor.max_check_first_iteration = seconds_from_start
        return True

    def execute_remotely(self, queue_name=None, clone=False, exit_process=True):
        # type: (Optional[str], bool, bool) -> Optional[Task]
        """
        If task is running locally (i.e., not by ``clearml-agent``), then clone the Task and enqueue it for remote
        execution; or, stop the execution of the current Task, reset its state, and enqueue it. If ``exit==True``,
        *exit* this process.

        .. note::
            If the task is running remotely (i.e., ``clearml-agent`` is executing it), this call is a no-op
            (i.e., does nothing).

        :param queue_name: The queue name used for enqueueing the task. If ``None``, this call exits the process
            without enqueuing the task.
        :param clone: Clone the Task and execute the newly cloned Task

            The values are:

            - ``True`` - A cloned copy of the Task will be created, and enqueued, instead of this Task.
            - ``False`` - The Task will be enqueued.

        :param exit_process: The function call will leave the calling process at the end

            - ``True`` - Exit the process (exit(0)).
            - ``False`` - Do not exit the process.

            .. warning::

                If ``clone==False``, then ``exit_process`` must be ``True``.

        :return Task: return the task object of the newly generated remotely executing task
        """
        # do nothing, we are running remotely
        if running_remotely() and self.is_main_task():
            return None

        if not self.is_main_task():
            LoggerRoot.get_base_logger().warning(
                "Calling task.execute_remotely is only supported on main Task (created with Task.init)\n"
                "Defaulting to self.enqueue(queue_name={})".format(queue_name)
            )
            if not queue_name:
                raise ValueError("queue_name must be provided")
            enqueue_task = Task.clone(source_task=self) if clone else self
            Task.enqueue(task=enqueue_task, queue_name=queue_name)
            return

        if not clone and not exit_process:
            raise ValueError(
                "clone==False and exit_process==False is not supported. "
                "Task enqueuing itself must exit the process afterwards.")

        # make sure we analyze the process
        if self.status in (Task.TaskStatusEnum.in_progress,):
            if clone:
                # wait for repository detection (5 minutes should be reasonable time to detect all packages)
                self.flush(wait_for_uploads=True)
                if self._logger and not self.__is_subprocess():
                    self._wait_for_repo_detection(timeout=300.)
            else:
                # close ourselves (it will make sure the repo is updated)
                self.close()

        # clone / reset Task
        if clone:
            task = Task.clone(self)
        else:
            task = self
            # check if the server supports enqueueing aborted/stopped Tasks
            if Session.check_min_api_server_version('2.13'):
                self.mark_stopped(force=True)
            else:
                self.reset()

        # enqueue ourselves
        if queue_name:
            Task.enqueue(task, queue_name=queue_name)
            LoggerRoot.get_base_logger().warning(
                'Switching to remote execution, output log page {}'.format(task.get_output_log_web_page()))
        else:
            # Remove the development system tag
            system_tags = [t for t in task.get_system_tags() if t != self._development_tag]
            self.set_system_tags(system_tags)
            # if we leave the Task out there, it makes sense to make it editable.
            self.reset(force=True)

        # leave this process.
        if exit_process:
            LoggerRoot.get_base_logger().warning('Terminating local execution process')
            leave_process(0)

        return task

    def create_function_task(self, func, func_name=None, task_name=None, **kwargs):
        # type: (Callable, Optional[str], Optional[str], **Optional[Any]) -> Optional[Task]
        """
        Create a new task, and call ``func`` with the specified kwargs.
        One can think of this call as remote forking, where the newly created instance is the new Task
        calling the specified func with the appropriate kwargs and leave once the func terminates.
        Notice that a remote executed function cannot create another child remote executed function.

        .. note::
            - Must be called from the main Task, i.e. the one created by Task.init(...)
            - The remote Tasks inherits the environment from the creating Task
            - In the remote Task, the entrypoint is the same as the creating Task
            - In the remote Task, the execution is the same until reaching this function call

        :param func: A function to execute remotely as a single Task.
            On the remote executed Task the entry-point and the environment are copied from this
            calling process, only this function call redirect the execution flow to the called func,
            alongside the passed arguments
        :param func_name: A unique identifier of the function. Default the function name without the namespace.
            For example Class.foo() becomes 'foo'
        :param task_name: The newly created Task name. Default: the calling Task name + function name
        :param kwargs: name specific arguments for the target function.
            These arguments will appear under the configuration, "Function" section

        :return Task: Return the newly created Task or None if running remotely and execution is skipped
        """
        if not self.is_main_task():
            raise ValueError("Only the main Task object can call create_function_task()")
        if not callable(func):
            raise ValueError("func must be callable")
        if not Session.check_min_api_version('2.9'):
            raise ValueError("Remote function execution is not supported, "
                             "please upgrade to the latest server version")

        func_name = str(func_name or func.__name__).strip()
        if func_name in self._remote_functions_generated:
            raise ValueError("Function name must be unique, a function by the name '{}' "
                             "was already created by this Task.".format(func_name))

        section_name = 'Function'
        tag_name = 'func'
        func_marker = '__func_readonly__'

        # sanitize the dict, leave only basic types that we might want to override later in the UI
        func_params = {k: v for k, v in kwargs.items() if verify_basic_value(v)}
        func_params[func_marker] = func_name

        # do not query if we are running locally, there is no need.
        task_func_marker = self.running_locally() or self.get_parameter('{}/{}'.format(section_name, func_marker))

        # if we are running locally or if we are running remotely but we are not a forked tasks
        # condition explained:
        # (1) running in development mode creates all the forked tasks
        # (2) running remotely but this is not one of the forked tasks (i.e. it is missing the fork tag attribute)
        if self.running_locally() or not task_func_marker:
            self._wait_for_repo_detection(300)
            task = self.clone(self, name=task_name or '{} <{}>'.format(self.name, func_name), parent=self.id)
            task.set_system_tags((task.get_system_tags() or []) + [tag_name])
            task.connect(func_params, name=section_name)
            self._remote_functions_generated[func_name] = task.id
            return task

        # check if we are one of the generated functions and if this is us,
        # if we are not the correct function, not do nothing and leave
        if task_func_marker != func_name:
            self._remote_functions_generated[func_name] = len(self._remote_functions_generated) + 1
            return

        # mark this is us:
        self._remote_functions_generated[func_name] = self.id

        # this is us for sure, let's update the arguments and call the function
        self.connect(func_params, name=section_name)
        func_params.pop(func_marker, None)
        kwargs.update(func_params)
        func(**kwargs)
        # This is it, leave the process
        leave_process(0)

    def wait_for_status(
            self,
            status=(_Task.TaskStatusEnum.completed, _Task.TaskStatusEnum.stopped, _Task.TaskStatusEnum.closed),
            raise_on_status=(_Task.TaskStatusEnum.failed,),
            check_interval_sec=60.,
    ):
        # type: (Iterable[Task.TaskStatusEnum], Optional[Iterable[Task.TaskStatusEnum]], float) -> ()
        """
        Wait for a task to reach a defined status.

        :param status: Status to wait for. Defaults to ('completed', 'stopped', 'closed', )
        :param raise_on_status: Raise RuntimeError if the status of the tasks matches one of these values.
            Defaults to ('failed').
        :param check_interval_sec: Interval in seconds between two checks. Defaults to 60 seconds.

        :raise: RuntimeError if the status is one of {raise_on_status}.
        """
        stopped_status = list(status) + (list(raise_on_status) if raise_on_status else [])
        while self.status not in stopped_status:
            time.sleep(check_interval_sec)

        if raise_on_status and self.status in raise_on_status:
            raise RuntimeError("Task {} has status: {}.".format(self.task_id, self.status))

        # make sure we have the Task object
        self.reload()

    def export_task(self):
        # type: () -> dict
        """
        Export Task's configuration into a dictionary (for serialization purposes).
        A Task can be copied/modified by calling Task.import_task()
        Notice: Export task does not include the tasks outputs, such as results
        (scalar/plots etc.) or Task artifacts/models

        :return: dictionary of the Task's configuration.
        """
        self.reload()
        export_data = self.data.to_dict()
        export_data.pop('last_metrics', None)
        export_data.pop('last_iteration', None)
        export_data.pop('status_changed', None)
        export_data.pop('status_reason', None)
        export_data.pop('status_message', None)
        export_data.get('execution', {}).pop('artifacts', None)
        export_data.get('execution', {}).pop('model', None)
        export_data['project_name'] = self.get_project_name()
        export_data['session_api_version'] = self.session.api_version
        return export_data

    def update_task(self, task_data):
        # type: (dict) -> bool
        """
        Update current task with configuration found on the task_data dictionary.
        See also export_task() for retrieving Task configuration.

        :param task_data: dictionary with full Task configuration
        :return: return True if Task update was successful
        """
        return bool(self.import_task(task_data=task_data, target_task=self, update=True))

    def rename(self, new_name):
        # type: (str) -> bool
        """
        Rename this task

        :param new_name: The new name of this task

        :return: True if the rename was successful and False otherwise
        """
        result = bool(self._edit(name=new_name))
        self.reload()
        return result

    def move_to_project(self, new_project_id=None, new_project_name=None, system_tags=None):
        # type: (Optional[str], Optional[str], Optional[Sequence[str]]) -> bool
        """
        Move this task to another project

        :param new_project_id: The ID of the project the task should be moved to.
            Not required if `new_project_name` is passed.
        :param new_project_name: Name of the new project the task should be moved to.
            Not required if `new_project_id` is passed.
        :param system_tags: System tags for the project the task should be moved to.

        :return: True if the move was successful and False otherwise
        """
        new_project_id = get_or_create_project(
            self.session, project_name=new_project_name, project_id=new_project_id, system_tags=system_tags
        )
        result = bool(self._edit(project=new_project_id))
        self.reload()
        return result

    @classmethod
    def import_task(cls, task_data, target_task=None, update=False):
        # type: (dict, Optional[Union[str, Task]], bool) -> Optional[Task]
        """
        Import (create) Task from previously exported Task configuration (see Task.export_task)
        Can also be used to edit/update an existing Task (by passing `target_task` and `update=True`).

        :param task_data: dictionary of a Task's configuration
        :param target_task: Import task_data into an existing Task. Can be either task_id (str) or Task object.
        :param update: If True, merge task_data with current Task configuration.
        :return: return True if Task was imported/updated
        """

        # restore original API version (otherwise, we might not be able to restore the data correctly)
        force_api_version = task_data.get('session_api_version') or None
        original_api_version = Session.api_version
        original_force_max_api_version = Session.force_max_api_version
        if force_api_version:
            Session.force_max_api_version = str(force_api_version)

        if not target_task:
            project_name = task_data.get('project_name') or Task._get_project_name(task_data.get('project', ''))
            target_task = Task.create(project_name=project_name, task_name=task_data.get('name', None))
        elif isinstance(target_task, six.string_types):
            target_task = Task.get_task(task_id=target_task)
        elif not isinstance(target_task, Task):
            raise ValueError(
                "`target_task` must be either Task id (str) or Task object, "
                "received `target_task` type {}".format(type(target_task)))
        target_task.reload()
        cur_data = target_task.data.to_dict()
        cur_data = merge_dicts(cur_data, task_data) if update else dict(**task_data)
        cur_data.pop('id', None)
        cur_data.pop('project', None)
        # noinspection PyProtectedMember
        valid_fields = list(tasks.EditRequest._get_data_props().keys())
        cur_data = dict((k, cur_data[k]) for k in valid_fields if k in cur_data)
        res = target_task._edit(**cur_data)
        if res and res.ok():
            target_task.reload()
        else:
            target_task = None

        # restore current api version, and return a new instance if Task with the current version
        if force_api_version:
            Session.force_max_api_version = original_force_max_api_version
            Session.api_version = original_api_version
            if target_task:
                target_task = Task.get_task(task_id=target_task.id)

        return target_task

    @classmethod
    def import_offline_session(cls, session_folder_zip, previous_task_id=None, iteration_offset=0):
        # type: (str, Optional[str], Optional[int]) -> (Optional[str])
        """
        Upload an offline session (execution) of a Task.
        Full Task execution includes repository details, installed packages, artifacts, logs, metric and debug samples.
        This function may also be used to continue a previously executed task with a task executed offline.

        :param session_folder_zip: Path to a folder containing the session, or zip-file of the session folder.
        :param previous_task_id: Task ID of the task you wish to continue with this offline session.
        :param iteration_offset: Reporting of the offline session will be offset with the
            number specified by this parameter. Useful for avoiding overwriting metrics.

        :return: Newly created task ID or the ID of the continued task (previous_task_id)
        """
        print('ClearML: Importing offline session from {}'.format(session_folder_zip))

        temp_folder = None
        if Path(session_folder_zip).is_file():
            # unzip the file:
            temp_folder = mkdtemp(prefix='clearml-offline-')
            ZipFile(session_folder_zip).extractall(path=temp_folder)
            session_folder_zip = temp_folder

        session_folder = Path(session_folder_zip)
        if not session_folder.is_dir():
            raise ValueError("Could not find the session folder / zip-file {}".format(session_folder))

        try:
            with open((session_folder / cls._offline_filename).as_posix(), 'rt') as f:
                export_data = json.load(f)
        except Exception as ex:
            raise ValueError(
                "Could not read Task object {}: Exception {}".format(session_folder / cls._offline_filename, ex))
        current_task = cls.import_task(export_data)
        if previous_task_id:
            task_holding_reports = cls.get_task(task_id=previous_task_id)
            task_holding_reports.mark_started(force=True)
            task_holding_reports = cls.import_task(export_data, target_task=task_holding_reports, update=True)
        else:
            task_holding_reports = current_task
            task_holding_reports.mark_started(force=True)
        # fix artifacts
        if current_task.data.execution.artifacts:
            from . import StorageManager
            # noinspection PyProtectedMember
            offline_folder = os.path.join(export_data.get('offline_folder', ''), 'data/')

            # noinspection PyProtectedMember
            remote_url = current_task._get_default_report_storage_uri()
            if remote_url and remote_url.endswith('/'):
                remote_url = remote_url[:-1]

            for artifact in current_task.data.execution.artifacts:
                local_path = artifact.uri.replace(offline_folder, '', 1)
                local_file = session_folder / 'data' / local_path
                if local_file.is_file():
                    remote_path = local_path.replace(
                        '.{}{}'.format(export_data['id'], os.sep), '.{}{}'.format(current_task.id, os.sep), 1)
                    artifact.uri = '{}/{}'.format(remote_url, remote_path)
                    StorageManager.upload_file(local_file=local_file.as_posix(), remote_url=artifact.uri)
            # noinspection PyProtectedMember
            task_holding_reports._edit(execution=current_task.data.execution)
        # logs
        TaskHandler.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset)
        # metrics
        Metrics.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset)
        # print imported results page
        print('ClearML results page: {}'.format(task_holding_reports.get_output_log_web_page()))
        task_holding_reports.mark_completed()
        # close task
        task_holding_reports.close()

        # cleanup
        if temp_folder:
            # noinspection PyBroadException
            try:
                shutil.rmtree(temp_folder)
            except Exception:
                pass

        return task_holding_reports.id

    @classmethod
    def set_credentials(
            cls,
            api_host=None,
            web_host=None,
            files_host=None,
            key=None,
            secret=None,
            store_conf_file=False
    ):
        # type: (Optional[str], Optional[str], Optional[str], Optional[str], Optional[str], bool) -> None
        """
        Set new default **ClearML Server** (backend) host and credentials.

        These credentials will be overridden by either OS environment variables, or the ClearML configuration
        file, ``clearml.conf``.

        .. warning::
           Credentials must be set before initializing a Task object.

        For example, to set credentials for a remote computer:

        .. code-block:: py

            Task.set_credentials(
                api_host='http://localhost:8008', web_host='http://localhost:8080', files_host='http://localhost:8081',
                key='optional_credentials',  secret='optional_credentials'
            )
            task = Task.init('project name', 'experiment name')

        :param str api_host: The API server url. For example, ``host='http://localhost:8008'``
        :param str web_host: The Web server url. For example, ``host='http://localhost:8080'``
        :param str files_host: The file server url. For example, ``host='http://localhost:8081'``
        :param str key: The user key (in the key/secret pair). For example, ``key='thisisakey123'``
        :param str secret: The user secret (in the key/secret pair). For example, ``secret='thisisseceret123'``
        :param bool store_conf_file: If True store the current configuration into the ~/clearml.conf file.
            If the configuration file exists, no change will be made (outputs a warning).
            Not applicable when running remotely (i.e. clearml-agent).
        """
        if api_host:
            Session.default_host = api_host
            if not running_remotely() and not ENV_HOST.get():
                ENV_HOST.set(api_host)
        if web_host:
            Session.default_web = web_host
            if not running_remotely() and not ENV_WEB_HOST.get():
                ENV_WEB_HOST.set(web_host)
        if files_host:
            Session.default_files = files_host
            if not running_remotely() and not ENV_FILES_HOST.get():
                ENV_FILES_HOST.set(files_host)
        if key:
            Session.default_key = key
            if not running_remotely():
                ENV_ACCESS_KEY.set(key)
        if secret:
            Session.default_secret = secret
            if not running_remotely():
                ENV_SECRET_KEY.set(secret)

        if store_conf_file and not running_remotely():
            active_conf_file = get_active_config_file()
            if active_conf_file:
                getLogger().warning(
                    'Could not store credentials in configuration file, '
                    '\'{}\' already exists'.format(active_conf_file))
            else:
                conf = {'api': dict(
                    api_server=Session.default_host,
                    web_server=Session.default_web,
                    files_server=Session.default_files,
                    credentials=dict(access_key=Session.default_key, secret_key=Session.default_secret))}
                with open(get_config_file(), 'wt') as f:
                    lines = json.dumps(conf, indent=4).split('\n')
                    f.write('\n'.join(lines[1:-1]))

    @classmethod
    def debug_simulate_remote_task(cls, task_id, reset_task=False):
        # type: (str, bool) -> ()
        """
        Simulate remote execution of a specified Task.
        This call will simulate the behaviour of your Task as if executed by the ClearML-Agent
        This means configurations will be coming from the backend server into the code
        (the opposite from manual execution, where the backend logs the code arguments)
        Use with care.

        :param task_id: Task ID to simulate, notice that all configuration will be taken from the specified
            Task, regardless of the code initial values, just like it as if executed by ClearML agent
        :param reset_task: If True target Task, is automatically cleared / reset.
        """

        # if we are already running remotely, do nothing
        if running_remotely():
            return

        # verify Task ID exists
        task = Task.get_task(task_id=task_id)
        if not task:
            raise ValueError("Task ID '{}' could not be found".format(task_id))

        if reset_task:
            task.reset(set_started_on_success=False, force=True)

        from .config.remote import override_current_task_id
        from .config.defs import LOG_TO_BACKEND_ENV_VAR
        override_current_task_id(task_id)
        LOG_TO_BACKEND_ENV_VAR.set(True)
        DEBUG_SIMULATE_REMOTE_TASK.set(True)

    @classmethod
    def _create(cls, project_name=None, task_name=None, task_type=TaskTypes.training):
        # type: (Optional[str], Optional[str], Task.TaskTypes) -> Task
        """
        Create a new unpopulated Task (experiment).

        :param str project_name: The name of the project in which the experiment will be created.
            If ``project_name`` is ``None``, and the main execution Task is initialized (see :meth:`Task.init`),
            then the main execution Task's project is used. Otherwise, if the project does
            not exist, it is created. (Optional)
        :param str task_name: The name of Task (experiment).
        :param TaskTypes task_type: The task type.

        :return: The newly created task created.
        """
        if not project_name:
            if not cls.__main_task:
                raise ValueError("Please provide project_name, no global task context found "
                                 "(Task.current_task hasn't been called)")
            project_name = cls.__main_task.get_project_name()

        try:
            task = cls(
                private=cls.__create_protection,
                project_name=project_name,
                task_name=task_name,
                task_type=task_type,
                log_to_backend=False,
                force_create=True,
            )
        except Exception:
            raise
        return task

    def _set_model_config(self, config_text=None, config_dict=None):
        # type: (Optional[str], Optional[Mapping]) -> None
        """
        Set Task model configuration text/dict

        :param config_text: model configuration (unconstrained text string). usually the content
            of a configuration file. If `config_text` is not None, `config_dict` must not be provided.
        :param config_dict: model configuration parameters dictionary.
            If `config_dict` is not None, `config_text` must not be provided.
        """
        # noinspection PyProtectedMember
        design = OutputModel._resolve_config(config_text=config_text, config_dict=config_dict)
        super(Task, self)._set_model_design(design=design)

    def _get_model_config_text(self):
        # type: () -> str
        """
        Get Task model configuration text (before creating an output model)
        When an output model is created it will inherit these properties

        :return: The model config_text (unconstrained text string).
        """
        return super(Task, self).get_model_design()

    def _get_model_config_dict(self):
        # type: () -> Dict
        """
        Get Task model configuration dictionary (before creating an output model)
        When an output model is created it will inherit these properties

        :return: config_dict: model configuration parameters dictionary.
        """
        config_text = self._get_model_config_text()
        # noinspection PyProtectedMember
        return OutputModel._text_to_config_dict(config_text)

    @classmethod
    def _reset_current_task_obj(cls):
        if not cls.__main_task:
            return
        task = cls.__main_task
        cls.__main_task = None
        if task._dev_worker:
            task._dev_worker.unregister()
            task._dev_worker = None

    @classmethod
    def _has_current_task_obj(cls):
        # type: () -> bool
        return bool(cls.__main_task)

    @classmethod
    def _create_dev_task(
            cls, default_project_name, default_task_name, default_task_type, tags,
            reuse_last_task_id, continue_last_task=False, detect_repo=True, auto_connect_streams=True
    ):
        if not default_project_name or not default_task_name:
            # get project name and task name from repository name and entry_point
            result, _ = ScriptInfo.get(create_requirements=False, check_uncommitted=False)
            if not default_project_name:
                # noinspection PyBroadException
                try:
                    parts = result.script['repository'].split('/')
                    default_project_name = (parts[-1] or parts[-2]).replace('.git', '') or 'Untitled'
                except Exception:
                    default_project_name = 'Untitled'
            if not default_task_name:
                # noinspection PyBroadException
                try:
                    default_task_name = os.path.splitext(os.path.basename(result.script['entry_point']))[0]
                except Exception:
                    pass

        # conform reuse_last_task_id and continue_last_task
        if continue_last_task and isinstance(continue_last_task, str):
            reuse_last_task_id = continue_last_task
            continue_last_task = True
        elif isinstance(continue_last_task, int) and continue_last_task is not True:
            # allow initial offset environment override
            continue_last_task = continue_last_task

        if TASK_SET_ITERATION_OFFSET.get() is not None:
            continue_last_task = TASK_SET_ITERATION_OFFSET.get()

        # if we force no task reuse from os environment
        if DEV_TASK_NO_REUSE.get() or not reuse_last_task_id or isinstance(reuse_last_task_id, str):
            default_task = None
        else:
            # if we have a previous session to use, get the task id from it
            default_task = cls.__get_last_used_task_id(
                default_project_name,
                default_task_name,
                default_task_type.value,
            )

        closed_old_task = False
        default_task_id = None
        task = None
        in_dev_mode = not running_remotely()

        if in_dev_mode:
            if isinstance(reuse_last_task_id, str) and reuse_last_task_id:
                default_task_id = reuse_last_task_id
            elif not reuse_last_task_id or not cls.__task_is_relevant(default_task):
                default_task_id = None
            else:
                default_task_id = default_task.get('id') if default_task else None

            if default_task_id:
                try:
                    task = cls(
                        private=cls.__create_protection,
                        task_id=default_task_id,
                        log_to_backend=True,
                    )

                    # instead of resting the previously used task we are continuing the training with it.
                    if task and \
                            (continue_last_task or
                             (isinstance(continue_last_task, int) and not isinstance(continue_last_task, bool))):
                        task.reload()
                        task.mark_started(force=True)
                        # allow to disable the
                        if continue_last_task is True:
                            task.set_initial_iteration(task.get_last_iteration() + 1)
                        else:
                            task.set_initial_iteration(continue_last_task)

                    else:
                        task_tags = task.data.system_tags if hasattr(task.data, 'system_tags') else task.data.tags
                        task_artifacts = task.data.execution.artifacts \
                            if hasattr(task.data.execution, 'artifacts') else None
                        if ((str(task._status) in (
                                str(tasks.TaskStatusEnum.published), str(tasks.TaskStatusEnum.closed)))
                                or task.output_models_id or (cls.archived_tag in task_tags)
                                or (cls._development_tag not in task_tags)
                                or task_artifacts):
                            # If the task is published or closed, we shouldn't reset it so we can't use it in dev mode
                            # If the task is archived, or already has an output model,
                            #  we shouldn't use it in development mode either
                            default_task_id = None
                            task = None
                        else:
                            with task._edit_lock:
                                # from now on, there is no need to reload, we just clear stuff,
                                # this flag will be cleared off once we actually refresh at the end of the function
                                task._reload_skip_flag = True
                                # reset the task, so we can update it
                                task.reset(set_started_on_success=False, force=False)
                                # clear the heaviest stuff first
                                task._clear_task(
                                    system_tags=[cls._development_tag],
                                    comment=make_message('Auto-generated at %(time)s by %(user)s@%(host)s'))

                except (Exception, ValueError):
                    # we failed reusing task, create a new one
                    default_task_id = None

        # create a new task
        if not default_task_id:
            task = cls(
                private=cls.__create_protection,
                project_name=default_project_name,
                task_name=default_task_name,
                task_type=default_task_type,
                log_to_backend=True,
            )
            # no need to reload yet, we clear this before the end of the function
            task._reload_skip_flag = True

        if in_dev_mode:
            # update this session, for later use
            cls.__update_last_used_task_id(default_project_name, default_task_name, default_task_type.value, task.id)
            # set default docker image from env.
            task._set_default_docker_image()

        # mark us as the main Task, there should only be one dev Task at a time.
        if not Task.__main_task:
            Task.__main_task = task

        # mark the task as started
        task.started()
        # reload, making sure we are synced
        task._reload_skip_flag = False
        task.reload()

        # add Task tags
        if tags:
            task.add_tags([tags] if isinstance(tags, str) else tags)

        # force update of base logger to this current task (this is the main logger task)
        logger = task._get_logger(auto_connect_streams=auto_connect_streams)
        if closed_old_task:
            logger.report_text('ClearML Task: Closing old development task id={}'.format(default_task.get('id')))
        # print warning, reusing/creating a task
        if default_task_id and not continue_last_task:
            logger.report_text('ClearML Task: overwriting (reusing) task id=%s' % task.id)
        elif default_task_id and continue_last_task:
            logger.report_text('ClearML Task: continuing previous task id=%s '
                               'Notice this run will not be reproducible!' % task.id)
        else:
            logger.report_text('ClearML Task: created new task id=%s' % task.id)

        # update current repository and put warning into logs
        if detect_repo:
            # noinspection PyBroadException
            try:
                import traceback
                stack = traceback.extract_stack(limit=10)
                # NOTICE WE ARE ALWAYS 3 down from caller in stack!
                for i in range(len(stack) - 1, 0, -1):
                    # look for the Task.init call, then the one above it is the callee module
                    if stack[i].name == 'init':
                        task._calling_filename = os.path.abspath(stack[i - 1].filename)
                        break
            except Exception:
                pass
            if in_dev_mode and cls.__detect_repo_async:
                task._detect_repo_async_thread = threading.Thread(target=task._update_repository)
                task._detect_repo_async_thread.daemon = True
                task._detect_repo_async_thread.start()
            else:
                task._update_repository()

        # make sure we see something in the UI
        thread = threading.Thread(target=LoggerRoot.flush)
        thread.daemon = True
        thread.start()

        return task

    def _get_logger(self, flush_period=NotSet, auto_connect_streams=False):
        # type: (Optional[float], Union[bool, dict]) -> Logger
        """
        get a logger object for reporting based on the task

        :param flush_period: The period of the logger flush.
            If None of any other False value, will not flush periodically.
            If a logger was created before, this will be the new period and
            the old one will be discarded.

        :return: Logger object
        """

        if not self._logger:
            # do not recreate logger after task was closed/quit
            if self._at_exit_called and self._at_exit_called in (True, get_current_thread_id(),):
                raise ValueError("Cannot use Task Logger after task was closed")
            # Get a logger object
            self._logger = Logger(
                private_task=self,
                connect_stdout=(auto_connect_streams is True) or
                               (isinstance(auto_connect_streams, dict) and auto_connect_streams.get('stdout', False)),
                connect_stderr=(auto_connect_streams is True) or
                               (isinstance(auto_connect_streams, dict) and auto_connect_streams.get('stderr', False)),
                connect_logging=isinstance(auto_connect_streams, dict) and auto_connect_streams.get('logging', False),
            )
            # make sure we set our reported to async mode
            # we make sure we flush it in self._at_exit
            self._reporter.async_enable = True
            # if we just created the logger, set default flush period
            if not flush_period or flush_period is self.NotSet:
                flush_period = float(DevWorker.report_period)

        if isinstance(flush_period, (int, float)):
            flush_period = int(abs(flush_period))

        if flush_period is None or isinstance(flush_period, int):
            self._logger.set_flush_period(flush_period)

        return self._logger

    def _connect_output_model(self, model, name=None):
        assert isinstance(model, OutputModel)
        model.connect(self, name=name)
        return model

    def _save_output_model(self, model):
        """
        Deprecated: Save a reference to the connected output model.

        :param model: The connected output model
        """
        # deprecated
        self._connected_output_model = model

    def _reconnect_output_model(self):
        """
        Deprecated: If there is a saved connected output model, connect it again.

        This is needed if the input model is connected after the output model
        is connected, an then we will have to get the model design from the
        input model by reconnecting.
        """
        # Deprecated:
        if self._connected_output_model:
            self.connect(self._connected_output_model)

    def _connect_input_model(self, model, name=None):
        assert isinstance(model, InputModel)
        # we only allow for an input model to be connected once
        # at least until we support multiple input models
        # notice that we do not check the task's input model because we allow task reuse and overwrite
        # add into comment that we are using this model
        comment = self.comment or ''
        if not comment.endswith('\n'):
            comment += '\n'
        comment += 'Using model id: {}'.format(model.id)
        self.set_comment(comment)

        model.connect(self, name)
        return model

    def _connect_argparse(self, parser, args=None, namespace=None, parsed_args=None, name=None):
        # do not allow argparser to connect to jupyter notebook
        # noinspection PyBroadException
        try:
            if 'IPython' in sys.modules:
                # noinspection PyPackageRequirements
                from IPython import get_ipython  # noqa
                ip = get_ipython()
                if ip is not None and 'IPKernelApp' in ip.config:
                    return parser
        except Exception:
            pass

        if self.is_main_task():
            argparser_update_currenttask(self)

        if (parser is None or parsed_args is None) and argparser_parseargs_called():
            # if we have a parser but nor parsed_args, we need to find the parser
            if parser and not parsed_args:
                for _parser, _parsed_args in get_argparser_last_args():
                    if _parser == parser:
                        parsed_args = _parsed_args
                        break
            else:
                # prefer the first argparser (hopefully it is more relevant?!
                for _parser, _parsed_args in get_argparser_last_args():
                    if parser is None:
                        parser = _parser
                    if parsed_args is None and parser == _parser:
                        parsed_args = _parsed_args

        if running_remotely() and (self.is_main_task() or self._is_remote_main_task()):
            self._arguments.copy_to_parser(parser, parsed_args)
        else:
            self._arguments.copy_defaults_from_argparse(
                parser, args=args, namespace=namespace, parsed_args=parsed_args)
        return parser

    def _connect_dictionary(self, dictionary, name=None):
        def _update_args_dict(task, config_dict):
            # noinspection PyProtectedMember
            task._arguments.copy_from_dict(flatten_dictionary(config_dict), prefix=name)

        def _refresh_args_dict(task, config_dict):
            # reread from task including newly added keys
            # noinspection PyProtectedMember
            a_flat_dict = task._arguments.copy_to_dict(flatten_dictionary(config_dict), prefix=name)
            # noinspection PyProtectedMember
            nested_dict = config_dict._to_dict()
            config_dict.clear()
            config_dict.update(nested_from_flat_dictionary(nested_dict, a_flat_dict))

        if not running_remotely() or not (self.is_main_task() or self._is_remote_main_task()):
            self._arguments.copy_from_dict(flatten_dictionary(dictionary), prefix=name)
            dictionary = ProxyDictPostWrite(self, _update_args_dict, **dictionary)
        else:
            flat_dict = flatten_dictionary(dictionary)
            flat_dict = self._arguments.copy_to_dict(flat_dict, prefix=name)
            dictionary = nested_from_flat_dictionary(dictionary, flat_dict)
            dictionary = ProxyDictPostWrite(self, _refresh_args_dict, **dictionary)

        return dictionary

    def _connect_task_parameters(self, attr_class, name=None):
        if running_remotely() and (self.is_main_task() or self._is_remote_main_task()):
            parameters = self.get_parameters()
            if not name:
                attr_class.update_from_dict(parameters)
            else:
                attr_class.update_from_dict(
                    dict((k[len(name) + 1:], v) for k, v in parameters.items() if k.startswith('{}/'.format(name))))
        else:
            self.set_parameters(attr_class.to_dict(), __parameters_prefix=name)
        return attr_class

    def _connect_object(self, an_object, name=None):
        def verify_type(key, value):
            if str(key).startswith('_') or not isinstance(value, self._parameters_allowed_types):
                return False
            # verify everything is json able (i.e. basic types)
            try:
                json.dumps(value)
                return True
            except TypeError:
                return False

        a_dict = {
            k: v
            for cls_ in getattr(an_object, "__mro__", [an_object])
            for k, v in cls_.__dict__.items()
            if verify_type(k, v)
        }
        if running_remotely() and (self.is_main_task() or self._is_remote_main_task()):
            a_dict = self._connect_dictionary(a_dict, name)
            for k, v in a_dict.items():
                if getattr(an_object, k, None) != a_dict[k]:
                    setattr(an_object, k, v)

            return an_object
        else:
            self._connect_dictionary(a_dict, name)
            return an_object

    def _dev_mode_stop_task(self, stop_reason, pid=None):
        # make sure we do not get called (by a daemon thread) after at_exit
        if self._at_exit_called:
            return

        self.log.warning(
            "### TASK STOPPED - USER ABORTED - {} ###".format(
                stop_reason.upper().replace('_', ' ')
            )
        )
        self.flush(wait_for_uploads=True)
        self.stopped(status_reason='USER ABORTED')

        if self._dev_worker:
            self._dev_worker.unregister()

        # NOTICE! This will end the entire execution tree!
        if self.__exit_hook:
            self.__exit_hook.remote_user_aborted = True
        self._kill_all_child_processes(send_kill=False, pid=pid, allow_kill_calling_pid=False)
        time.sleep(2.0)
        self._kill_all_child_processes(send_kill=True, pid=pid, allow_kill_calling_pid=True)
        os._exit(1)  # noqa

    @staticmethod
    def _kill_all_child_processes(send_kill=False, pid=None, allow_kill_calling_pid=True):
        # get current process if pid not provided
        current_pid = os.getpid()
        kill_ourselves = None
        pid = pid or current_pid
        try:
            parent = psutil.Process(pid)
        except psutil.Error:
            # could not find parent process id
            return
        for child in parent.children(recursive=True):
            # kill ourselves last (if we need to)
            if child.pid == current_pid:
                kill_ourselves = child
                continue
            if send_kill:
                child.kill()
            else:
                child.terminate()

        # parent ourselves
        if allow_kill_calling_pid or parent.pid != current_pid:
            if send_kill:
                parent.kill()
            else:
                parent.terminate()

        # kill ourselves if we need to:
        if allow_kill_calling_pid and kill_ourselves:
            if send_kill:
                kill_ourselves.kill()
            else:
                kill_ourselves.terminate()

    def _dev_mode_setup_worker(self):
        if (running_remotely() and not DEBUG_SIMULATE_REMOTE_TASK.get()) \
                or not self.is_main_task() or self._at_exit_called or self._offline_mode:
            return

        if self._dev_worker:
            return self._dev_worker

        self._dev_worker = DevWorker()
        self._dev_worker.register(self)

        logger = self.get_logger()
        flush_period = logger.get_flush_period()
        if not flush_period or flush_period > self._dev_worker.report_period:
            logger.set_flush_period(self._dev_worker.report_period)

    def _wait_for_repo_detection(self, timeout=None):
        # wait for detection repo sync
        if not self._detect_repo_async_thread:
            return
        with self._repo_detect_lock:
            if not self._detect_repo_async_thread:
                return
            # noinspection PyBroadException
            try:
                if self._detect_repo_async_thread.is_alive():
                    # if negative timeout, just kill the thread:
                    if timeout is not None and timeout < 0:
                        from .utilities.lowlevel.threads import kill_thread
                        kill_thread(self._detect_repo_async_thread)
                    else:
                        self.log.info('Waiting for repository detection and full package requirement analysis')
                        self._detect_repo_async_thread.join(timeout=timeout)
                        # because join has no return value
                        if self._detect_repo_async_thread.is_alive():
                            self.log.info('Repository and package analysis timed out ({} sec), '
                                          'giving up'.format(timeout))
                            # done waiting, kill the thread
                            from .utilities.lowlevel.threads import kill_thread
                            kill_thread(self._detect_repo_async_thread)
                        else:
                            self.log.info('Finished repository detection and package analysis')
                self._detect_repo_async_thread = None
            except Exception:
                pass

    def _summary_artifacts(self):
        # signal artifacts upload, and stop daemon
        self._artifacts_manager.stop(wait=True)
        # print artifacts summary (if not empty)
        if self._artifacts_manager.summary:
            self.get_logger().report_text(self._artifacts_manager.summary)

    def _at_exit(self):
        # protect sub-process at_exit (should never happen)
        if self._at_exit_called and self._at_exit_called != get_current_thread_id():
            return

        # make sure we do not try to use events, because Python might deadlock itself.
        # https://bugs.python.org/issue41606
        if self.__is_subprocess():
            BackgroundMonitor.set_at_exit_state(True)

        # shutdown will clear the main, so we have to store it before.
        # is_main = self.is_main_task()
        # fix debugger signal in the middle, catch everything
        try:
            self.__shutdown()
        except:  # noqa
            pass
        # In rare cases we might need to forcefully shutdown the process, currently we should avoid it.
        # if is_main:
        #     # we have to forcefully shutdown if we have forked processes, sometimes they will get stuck
        #     os._exit(self.__exit_hook.exit_code if self.__exit_hook and self.__exit_hook.exit_code else 0)

    def __shutdown(self):
        """
        Will happen automatically once we exit code, i.e. atexit
        :return:
        """
        # protect sub-process at_exit
        if self._at_exit_called:
            is_sub_process = self.__is_subprocess()
            # if we are called twice (signal in the middle of the shutdown),
            _nested_shutdown_call = bool(self._at_exit_called == get_current_thread_id())
            if _nested_shutdown_call and not is_sub_process:
                # if we were called again in the main thread on the main process, let's try again
                # make sure we only do this once
                self._at_exit_called = True
            else:
                # make sure we flush stdout, this is the best we can do.
                if _nested_shutdown_call and self._logger and is_sub_process:
                    # noinspection PyProtectedMember
                    self._logger._close_stdout_handler(wait=True)
                    self._at_exit_called = True
                # if we get here, we should do nothing and leave
                return
        else:
            # from here only a single thread can re-enter
            self._at_exit_called = get_current_thread_id()

        # disable lock on signal callbacks, to avoid deadlocks.
        if self.__exit_hook and self.__exit_hook.signal is not None:
            self.__edit_lock = False

        is_sub_process = self.__is_subprocess()

        task_status = None
        # noinspection PyBroadException
        try:
            wait_for_uploads = True
            # first thing mark task as stopped, so we will not end up with "running" on lost tasks
            # if we are running remotely, the daemon will take care of it
            wait_for_std_log = True
            if (not running_remotely() or DEBUG_SIMULATE_REMOTE_TASK.get()) \
                    and self.is_main_task() and not is_sub_process:
                # check if we crashed, ot the signal is not interrupt (manual break)
                task_status = ('stopped',)
                if self.__exit_hook:
                    is_exception = self.__exit_hook.exception
                    # check if we are running inside a debugger
                    if not is_exception and sys.modules.get('pydevd'):
                        # noinspection PyBroadException
                        try:
                            is_exception = sys.last_type
                        except Exception:
                            pass

                        # check if this is Jupyter interactive session, do not mark as exception
                        if 'IPython' in sys.modules:
                            is_exception = None

                    # only if we have an exception (and not ctrl-break) or signal is not SIGTERM / SIGINT
                    if (is_exception and not isinstance(is_exception, KeyboardInterrupt)
                        and is_exception != KeyboardInterrupt) \
                            or (not self.__exit_hook.remote_user_aborted and
                                (self.__exit_hook.signal not in (None, 2, 15) or self.__exit_hook.exit_code)):
                        task_status = (
                            'failed',
                            'Exception {}'.format(is_exception) if is_exception else
                            'Signal {}'.format(self.__exit_hook.signal))
                        wait_for_uploads = False
                    else:
                        wait_for_uploads = (self.__exit_hook.remote_user_aborted or self.__exit_hook.signal is None)
                        if not self.__exit_hook.remote_user_aborted and self.__exit_hook.signal is None and \
                                not is_exception:
                            task_status = ('completed',)
                        else:
                            task_status = ('stopped',)
                            # user aborted. do not bother flushing the stdout logs
                            wait_for_std_log = self.__exit_hook.signal is not None

            # wait for repository detection (if we didn't crash)
            if wait_for_uploads and self._logger:
                # we should print summary here
                self._summary_artifacts()
                # make sure that if we crashed the thread we are not waiting forever
                if not is_sub_process:
                    self._wait_for_repo_detection(timeout=10.)

            # kill the repo thread (negative timeout, do not wait), if it hasn't finished yet.
            if not is_sub_process:
                self._wait_for_repo_detection(timeout=-1)

            # wait for uploads
            print_done_waiting = False
            if wait_for_uploads and (BackendModel.get_num_results() > 0 or
                                     (self.__reporter and self.__reporter.events_waiting())):
                self.log.info('Waiting to finish uploads')
                print_done_waiting = True
            # from here, do not send log in background thread
            if wait_for_uploads:
                self.flush(wait_for_uploads=True)
                # wait until the reporter flush everything
                if self.__reporter:
                    self.__reporter.stop()
                    if self.is_main_task():
                        # notice: this will close the reporting for all the Tasks in the system
                        Metrics.close_async_threads()
                        # notice: this will close the jupyter monitoring
                        ScriptInfo.close()
                if self.is_main_task():
                    # noinspection PyBroadException
                    try:
                        from .storage.helper import StorageHelper
                        StorageHelper.close_async_threads()
                    except Exception:
                        pass

                if print_done_waiting:
                    self.log.info('Finished uploading')
            # elif self._logger:
            #     # noinspection PyProtectedMember
            #     self._logger._flush_stdout_handler()

            # from here, do not check worker status
            if self._dev_worker:
                self._dev_worker.unregister()
                self._dev_worker = None

            # stop resource monitoring
            if self._resource_monitor:
                self._resource_monitor.stop()
                self._resource_monitor = None

            if self._logger:
                self._logger.set_flush_period(None)
                # noinspection PyProtectedMember
                self._logger._close_stdout_handler(wait=wait_for_uploads or wait_for_std_log)

            if not is_sub_process:
                # change task status
                if not task_status:
                    pass
                elif task_status[0] == 'failed':
                    self.mark_failed(status_reason=task_status[1])
                elif task_status[0] == 'completed':
                    self.set_progress(100)
                    self.mark_completed()
                elif task_status[0] == 'stopped':
                    self.stopped()

            # this is so in theory we can close a main task and start a new one
            if self.is_main_task():
                Task.__main_task = None
                Task.__update_master_pid_task(task=None)
        except Exception:
            # make sure we do not interrupt the exit process
            pass

        # make sure we store last task state
        if self._offline_mode and not is_sub_process:
            # noinspection PyBroadException
            try:
                # create zip file
                offline_folder = self.get_offline_mode_folder()
                zip_file = offline_folder.as_posix() + '.zip'
                with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf:
                    for filename in offline_folder.rglob('*'):
                        if filename.is_file():
                            relative_file_name = filename.relative_to(offline_folder).as_posix()
                            zf.write(filename.as_posix(), arcname=relative_file_name)
                print('ClearML Task: Offline session stored in {}'.format(zip_file))
            except Exception:
                pass

        # delete locking object (lock file)
        if self._edit_lock:
            # noinspection PyBroadException
            try:
                del self.__edit_lock
            except Exception:
                pass
            self._edit_lock = None

        # make sure no one will re-enter the shutdown method
        self._at_exit_called = True
        if not is_sub_process and BackgroundMonitor.is_subprocess_enabled():
            BackgroundMonitor.wait_for_sub_process(self)

    @classmethod
    def __register_at_exit(cls, exit_callback, only_remove_signal_and_exception_hooks=False):
        class ExitHooks(object):
            _orig_exit = None
            _orig_exc_handler = None
            remote_user_aborted = False

            def __init__(self, callback):
                self.exit_code = None
                self.exception = None
                self.signal = None
                self._exit_callback = callback
                self._org_handlers = {}
                self._signal_recursion_protection_flag = False
                self._except_recursion_protection_flag = False

            def update_callback(self, callback):
                if self._exit_callback and not six.PY2:
                    # noinspection PyBroadException
                    try:
                        atexit.unregister(self._exit_callback)
                    except Exception:
                        pass
                self._exit_callback = callback
                if callback:
                    self.hook()
                else:
                    # un register int hook
                    if self._orig_exc_handler:
                        sys.excepthook = self._orig_exc_handler
                        self._orig_exc_handler = None
                    for h in self._org_handlers:
                        # noinspection PyBroadException
                        try:
                            signal.signal(h, self._org_handlers[h])
                        except Exception:
                            pass
                    self._org_handlers = {}

            def hook(self):
                if self._orig_exit is None:
                    self._orig_exit = sys.exit
                    sys.exit = self.exit

                if self._orig_exc_handler is None:
                    self._orig_exc_handler = sys.excepthook
                    sys.excepthook = self.exc_handler

                if self._exit_callback:
                    atexit.register(self._exit_callback)

                # TODO: check if sub-process hooks are safe enough, for the time being allow it
                if not self._org_handlers:  # ## and not Task._Task__is_subprocess():
                    if sys.platform == 'win32':
                        catch_signals = [signal.SIGINT, signal.SIGTERM, signal.SIGSEGV, signal.SIGABRT,
                                         signal.SIGILL, signal.SIGFPE]
                    else:
                        catch_signals = [signal.SIGINT, signal.SIGTERM, signal.SIGSEGV, signal.SIGABRT,
                                         signal.SIGILL, signal.SIGFPE, signal.SIGQUIT]
                    for c in catch_signals:
                        # noinspection PyBroadException
                        try:
                            self._org_handlers[c] = signal.getsignal(c)
                            signal.signal(c, self.signal_handler)
                        except Exception:
                            pass

            def exit(self, code=0):
                self.exit_code = code
                self._orig_exit(code)

            def exc_handler(self, exctype, value, traceback, *args, **kwargs):
                if self._except_recursion_protection_flag:
                    # noinspection PyArgumentList
                    return sys.__excepthook__(exctype, value, traceback, *args, **kwargs)

                self._except_recursion_protection_flag = True
                self.exception = value
                if self._orig_exc_handler:
                    # noinspection PyArgumentList
                    ret = self._orig_exc_handler(exctype, value, traceback, *args, **kwargs)
                else:
                    # noinspection PyNoneFunctionAssignment, PyArgumentList
                    ret = sys.__excepthook__(exctype, value, traceback, *args, **kwargs)
                self._except_recursion_protection_flag = False

                return ret

            def signal_handler(self, sig, frame):
                self.signal = sig

                org_handler = self._org_handlers.get(sig)
                signal.signal(sig, org_handler or signal.SIG_DFL)

                # if this is a sig term, we wait until __at_exit is called (basically do nothing)
                if sig == signal.SIGINT:
                    # return original handler result
                    return org_handler if not callable(org_handler) else org_handler(sig, frame)

                if self._signal_recursion_protection_flag:
                    # call original
                    os.kill(os.getpid(), sig)
                    return org_handler if not callable(org_handler) else org_handler(sig, frame)

                self._signal_recursion_protection_flag = True

                # call exit callback
                if self._exit_callback:
                    # noinspection PyBroadException
                    try:
                        self._exit_callback()
                    except Exception:
                        pass

                # remove stdout logger, just in case
                # noinspection PyBroadException
                try:
                    # noinspection PyProtectedMember
                    Logger._remove_std_logger()
                except Exception:
                    pass

                # noinspection PyUnresolvedReferences
                os.kill(os.getpid(), sig)

                self._signal_recursion_protection_flag = False
                # return handler result
                return org_handler if not callable(org_handler) else org_handler(sig, frame)

        # we only remove the signals since this will hang subprocesses
        if only_remove_signal_and_exception_hooks:
            if not cls.__exit_hook:
                return
            if cls.__exit_hook._orig_exc_handler:
                sys.excepthook = cls.__exit_hook._orig_exc_handler
                cls.__exit_hook._orig_exc_handler = None
            for s in cls.__exit_hook._org_handlers:
                # noinspection PyBroadException
                try:
                    signal.signal(s, cls.__exit_hook._org_handlers[s])
                except Exception:
                    pass
            cls.__exit_hook._org_handlers = {}
            return

        if cls.__exit_hook is None:
            # noinspection PyBroadException
            try:
                cls.__exit_hook = ExitHooks(exit_callback)
                cls.__exit_hook.hook()
            except Exception:
                cls.__exit_hook = None
        else:
            cls.__exit_hook.update_callback(exit_callback)

    def _remove_at_exit_callbacks(self):
        self.__register_at_exit(None, only_remove_signal_and_exception_hooks=True)
        # noinspection PyProtectedMember
        atexit.unregister(self.__exit_hook._exit_callback)
        self._at_exit_called = True

    @classmethod
    def __get_task(
            cls,
            task_id=None,  # type: Optional[str]
            project_name=None,  # type: Optional[str]
            task_name=None,  # type: Optional[str]
            include_archived=True,  # type: bool
            tags=None,  # type: Optional[Sequence[str]]
            task_filter=None  # type: Optional[dict]
    ):
        # type: (...) -> Task

        if task_id:
            return cls(private=cls.__create_protection, task_id=task_id, log_to_backend=False)

        if project_name:
            res = cls._send(
                cls._get_default_session(),
                projects.GetAllRequest(
                    name=exact_match_regex(project_name)
                )
            )
            project = get_single_result(entity='project', query=project_name, results=res.response.projects)
        else:
            project = None

        # get default session, before trying to access tasks.Task so that we do not create two sessions.
        session = cls._get_default_session()
        system_tags = 'system_tags' if hasattr(tasks.Task, 'system_tags') else 'tags'
        task_filter = task_filter or {}
        if not include_archived:
            task_filter['system_tags'] = (task_filter.get('system_tags') or []) + ['-{}'.format(cls.archived_tag)]
        if tags:
            task_filter['tags'] = (task_filter.get('tags') or []) + list(tags)
        res = cls._send(
            session,
            tasks.GetAllRequest(
                project=[project.id] if project else None,
                name=exact_match_regex(task_name) if task_name else None,
                only_fields=['id', 'name', 'last_update', system_tags],
                **task_filter
            )
        )
        res_tasks = res.response.tasks
        # if we have more than one result, filter out the 'archived' results
        # notice that if we only have one result we do get the archived one as well.
        if len(res_tasks) > 1:
            filtered_tasks = [t for t in res_tasks if not getattr(t, system_tags, None) or
                              cls.archived_tag not in getattr(t, system_tags, None)]
            # if we did not filter everything (otherwise we have only archived tasks, so we return them)
            if filtered_tasks:
                res_tasks = filtered_tasks

        task = get_single_result(
            entity='task',
            query={k: v for k, v in dict(
                project_name=project_name, task_name=task_name, tags=tags,
                include_archived=include_archived, task_filter=task_filter).items() if v},
            results=res_tasks, raise_on_error=False)
        if not task:
            return None

        return cls(
            private=cls.__create_protection,
            task_id=task.id,
            log_to_backend=False,
        )

    @classmethod
    def __get_tasks(
        cls,
        task_ids=None,  # type: Optional[Sequence[str]]
        project_name=None,  # type: Optional[Union[Sequence[str],str]]
        task_name=None,  # type: Optional[str]
        **kwargs  # type: Any
    ):
        # type: (...) -> List[Task]

        if task_ids:
            if isinstance(task_ids, six.string_types):
                task_ids = [task_ids]
            return [cls(private=cls.__create_protection, task_id=task_id, log_to_backend=False) for task_id in task_ids]

        queried_tasks = cls._query_tasks(
            project_name=project_name, task_name=task_name, fetch_only_first_page=True, **kwargs
        )
        if len(queried_tasks) == 500:
            LoggerRoot.get_base_logger().warning(
                "Too many requests when calling Task.get_tasks()."
                " Returning only the first 500 results."
                " Use Task.query_tasks() to fetch all task IDs"
            )
        return [cls(private=cls.__create_protection, task_id=task.id, log_to_backend=False) for task in queried_tasks]

    @classmethod
    def _query_tasks(cls, task_ids=None, project_name=None, task_name=None, fetch_only_first_page=False, **kwargs):
        res = None
        if not task_ids:
            task_ids = None
        elif isinstance(task_ids, six.string_types):
            task_ids = [task_ids]

        if project_name and isinstance(project_name, str):
            project_names = [project_name]
        else:
            project_names = project_name

        project_ids = []
        if project_names:
            for name in project_names:
                aux_kwargs = {}
                if kwargs.get("_allow_extra_fields_"):
                    aux_kwargs["_allow_extra_fields_"] = True
                    aux_kwargs["search_hidden"] = kwargs.get("search_hidden", False)
                res = cls._send(
                    cls._get_default_session(),
                    projects.GetAllRequest(
                        name=exact_match_regex(name),
                        **aux_kwargs
                    )
                )
                project = get_single_result(entity='project', query=name, results=res.response.projects)
                if project:
                    project_ids.append(project.id)

        session = cls._get_default_session()
        system_tags = 'system_tags' if hasattr(tasks.Task, 'system_tags') else 'tags'
        only_fields = ['id', 'name', 'last_update', system_tags]

        if kwargs and kwargs.get('only_fields'):
            only_fields = list(set(kwargs.pop('only_fields')) | set(only_fields))

        # if we have specific page to look for, we should only get the requested one
        if not fetch_only_first_page and kwargs and 'page' in kwargs:
            fetch_only_first_page = True

        ret_tasks = []
        page = -1
        page_size = 500
        while page == -1 or (not fetch_only_first_page and res and len(res.response.tasks) == page_size):
            page += 1
            # work on a copy and make sure we override all fields with ours
            request_kwargs = dict(
                id=task_ids,
                project=project_ids if project_ids else kwargs.pop("project", None),
                name=task_name if task_name else kwargs.pop("name", None),
                only_fields=only_fields,
                page=page,
                page_size=page_size,
            )
            # make sure we always override with the kwargs (specifically page selection / page_size)
            request_kwargs.update(kwargs or {})
            res = cls._send(
                session,
                tasks.GetAllRequest(**request_kwargs),
            )
            ret_tasks.extend(res.response.tasks)
        return ret_tasks

    @classmethod
    def _wait_for_deferred(cls, task):
        # type: (Optional[Task]) -> None
        """
        Make sure the task object deferred `Task.init` is completed.
        Accessing any of the `task` object's property will ensure the Task.init call was also complete
        This is an internal utility function

        :param task: Optional deferred Task object as returned form Task.init
        """
        if not task:
            return
        # force deferred init to complete
        task.id  # noqa

    @classmethod
    def __get_hash_key(cls, *args):
        def normalize(x):
            return "<{}>".format(x) if x is not None else ""

        return ":".join(map(normalize, args))

    @classmethod
    def __get_last_used_task_id(cls, default_project_name, default_task_name, default_task_type):
        hash_key = cls.__get_hash_key(
            cls._get_api_server(), default_project_name, default_task_name, default_task_type)

        # check if we have a cached task_id we can reuse
        # it must be from within the last 24h and with the same project/name/type
        task_sessions = SessionCache.load_dict(str(cls))

        task_data = task_sessions.get(hash_key)
        if task_data is None:
            return None

        try:
            task_data['type'] = cls.TaskTypes(task_data['type'])
        except (ValueError, KeyError):
            LoggerRoot.get_base_logger().warning(
                "Corrupted session cache entry: {}. "
                "Unsupported task type: {}"
                "Creating a new task.".format(hash_key, task_data['type']),
            )

            return None

        return task_data

    @classmethod
    def __update_last_used_task_id(cls, default_project_name, default_task_name, default_task_type, task_id):
        hash_key = cls.__get_hash_key(
            cls._get_api_server(), default_project_name, default_task_name, default_task_type)

        task_id = str(task_id)
        # update task session cache
        task_sessions = SessionCache.load_dict(str(cls))
        last_task_session = {'time': time.time(), 'project': default_project_name, 'name': default_task_name,
                             'type': default_task_type, 'id': task_id}

        # remove stale sessions
        for k in list(task_sessions.keys()):
            if ((time.time() - task_sessions[k].get('time', 0)) >
                    60 * 60 * cls.__task_id_reuse_time_window_in_hours):
                task_sessions.pop(k)
        # update current session
        task_sessions[hash_key] = last_task_session
        # store
        SessionCache.store_dict(str(cls), task_sessions)

    @classmethod
    def __task_timed_out(cls, task_data):
        return \
            task_data and \
            task_data.get('id') and \
            task_data.get('time') and \
            (time.time() - task_data.get('time')) > (60 * 60 * cls.__task_id_reuse_time_window_in_hours)

    @classmethod
    def __get_task_api_obj(cls, task_id, only_fields=None):
        if not task_id or cls._offline_mode:
            return None

        all_tasks = cls._send(
            cls._get_default_session(),
            tasks.GetAllRequest(id=[task_id], only_fields=only_fields),
        ).response.tasks

        # The task may not exist in environment changes
        if not all_tasks:
            return None

        return all_tasks[0]

    @classmethod
    def __task_is_relevant(cls, task_data):
        """
        Check that a cached task is relevant for reuse.

        A task is relevant for reuse if:
            1. It is not timed out i.e it was last use in the previous 24 hours.
            2. It's name, project and type match the data in the server, so not
               to override user changes made by using the UI.

        :param task_data: A mapping from 'id', 'name', 'project', 'type' keys
            to the task's values, as saved in the cache.

        :return: True, if the task is relevant for reuse. False, if not.
        """
        if not task_data:
            return False

        if cls.__task_timed_out(task_data):
            return False

        task_id = task_data.get('id')

        if not task_id:
            return False

        # noinspection PyBroadException
        try:
            task = cls.__get_task_api_obj(task_id, ('id', 'name', 'project', 'type'))
        except Exception:
            task = None

        if task is None:
            return False

        project_name = None
        if task.project:
            # noinspection PyBroadException
            try:
                project = cls._send(
                    cls._get_default_session(),
                    projects.GetByIdRequest(project=task.project)
                ).response.project

                if project:
                    project_name = project.name
            except Exception:
                pass

        if task_data.get('type') and \
                task_data.get('type') not in (cls.TaskTypes.training, cls.TaskTypes.testing) and \
                not Session.check_min_api_version(2.8):
            print('WARNING: Changing task type to "{}" : '
                  'clearml-server does not support task type "{}", '
                  'please upgrade clearml-server.'.format(cls.TaskTypes.training, task_data['type'].value))
            task_data['type'] = cls.TaskTypes.training

        compares = (
            (task.name, 'name'),
            (project_name, 'project'),
            (task.type, 'type'),
        )

        # compare after casting to string to avoid enum instance issues
        # remember we might have replaced the api version by now, so enums are different
        return all(six.text_type(server_data) == six.text_type(task_data.get(task_data_key))
                   for server_data, task_data_key in compares)

    @classmethod
    def __close_timed_out_task(cls, task_data):
        if not task_data:
            return False

        task = cls.__get_task_api_obj(task_data.get('id'), ('id', 'status'))

        if task is None:
            return False

        stopped_statuses = (
            str(tasks.TaskStatusEnum.stopped),
            str(tasks.TaskStatusEnum.published),
            str(tasks.TaskStatusEnum.publishing),
            str(tasks.TaskStatusEnum.closed),
            str(tasks.TaskStatusEnum.failed),
            str(tasks.TaskStatusEnum.completed),
        )

        if str(task.status) not in stopped_statuses:
            cls._send(
                cls._get_default_session(),
                tasks.StoppedRequest(
                    task=task.id,
                    force=True,
                    status_message="Stopped timed out development task"
                ),
            )

            return True
        return False

    @classmethod
    def __add_model_wildcards(cls, auto_connect_frameworks):
        if isinstance(auto_connect_frameworks, dict):
            for k, v in auto_connect_frameworks.items():
                if isinstance(v, str):
                    v = [v]
                if isinstance(v, (list, tuple)):
                    WeightsFileHandler.model_wildcards[k] = [str(i) for i in v]

        def callback(_, model_info):
            if not model_info:
                return None
            parents = Framework.get_framework_parents(model_info.framework)
            wildcards = []
            for parent in parents:
                if WeightsFileHandler.model_wildcards.get(parent):
                    wildcards.extend(WeightsFileHandler.model_wildcards[parent])
            if not wildcards:
                return model_info
            if not matches_any_wildcard(model_info.local_model_path, wildcards):
                return None
            return model_info

        WeightsFileHandler.add_pre_callback(callback)

    def __getstate__(self):
        # type: () -> dict
        return {'main': self.is_main_task(), 'id': self.id, 'offline': self.is_offline()}

    def __setstate__(self, state):
        if state['main'] and not self.__main_task:
            Task.__forked_proc_main_pid = None
            Task.__update_master_pid_task(task=state['id'])
        if state['offline']:
            Task.set_offline(offline_mode=state['offline'])

        task = Task.init(
            continue_last_task=state['id'],
            auto_connect_frameworks={'detect_repository': False}) \
            if state['main'] else Task.get_task(task_id=state['id'])
        self.__dict__ = task.__dict__


# clearml/task_parameters.py

import six
import attr
from attr import validators


__all__ = ['range_validator', 'param', 'percent_param', 'TaskParameters']


def _canonize_validator(current_validator):
    """
    Convert current_validator to a new list and return it.

    If current_validator is None return an empty list.
    If current_validator is a list, return a copy of it.
    If current_validator is another type of  iterable, return a list version of it.
    If current_validator is a single value, return a one-list containing it.
    """

    if not current_validator:
        return []

    if isinstance(current_validator, (list, tuple)):
        current_validator = list(current_validator)
    else:
        current_validator = [current_validator]

    return current_validator


def range_validator(min_value, max_value):
    """
    A parameter validator that checks range constraint on a parameter.

    :param min_value: The minimum limit of the range, inclusive. None for no minimum limit.
    :param max_value: The maximum limit of the range, inclusive. None for no maximum limit.
    :return: A new range validator.
    """
    def _range_validator(instance, attribute, value):
        if ((min_value is not None) and (value < min_value)) or \
                ((max_value is not None) and (value > max_value)):
            raise ValueError("{} must be in range [{}, {}]".format(attribute.name, min_value, max_value))

    return _range_validator


def param(
        validator=None,
        range=None,
        type=None,
        desc=None,
        metadata=None,
        *args,
        **kwargs
):
    """
    A parameter inside a TaskParameters class.

    See TaskParameters for more information.

    :param validator: A validator or validators list.
        Any validator from attr.validators is applicable.

    :param range: The legal values range of the parameter.
        A tuple (min_limit, max_limit). None for no limitation.

    :param type: The type of the parameter.
        Supported types are int, str and float. None to place no limit of the type

    :param desc: A string description of the parameter, for future use.

    :param metadata: A dictionary metadata of the parameter, for future use.

    :param args: Additional arguments to pass to attr.attrib constructor.
    :param kwargs: Additional keyword arguments to pass to attr.attrib constructor.

    :return: An attr.attrib instance to use with TaskParameters class.

    Warning: Do not create an immutable param using args or kwargs. It will cause
    connect method of the TaskParameters class to fail.
    """

    metadata = metadata or {}
    metadata["desc"] = desc

    validator = _canonize_validator(validator)

    if type:
        validator.append(validators.optional(validators.instance_of(type)))

    if range:
        validator.append(range_validator(*range))

    return attr.ib(validator=validator, type=type, metadata=metadata, *args, **kwargs)


def percent_param(*args, **kwargs):
    """
    A param with type float and range limit (0, 1).
    """
    return param(range=(0, 1), type=float, *args, **kwargs)


class _AttrsMeta(type):
    def __new__(mcs, name, bases, dct):
        new_class = super(_AttrsMeta, mcs).__new__(mcs, name, bases, dct)
        return attr.s(new_class)


@six.add_metaclass(_AttrsMeta)
class TaskParameters(object):
    """
    Base class for task parameters.

    Inherit this class to create a parameter set to connect to a task.

    Usage Example:
    class MyParams(TaskParameters):
        iterations = param(
            type=int,
            desc="Number of iterations to run",
            range=(0, 100000),
        )

        target_accuracy = percent_param(
            desc="The target accuracy of the model",
        )
    """

    def to_dict(self):
        """
        :return: A new dictionary with keys are the parameters names and values
            are the corresponding values.
        """
        return attr.asdict(self)

    def update_from_dict(self, source_dict):
        """
        Update the parameters using values from a dictionary.

        :param source_dict: A dictionary with an entry for each parameter to
            update.
        """
        for key, value in source_dict.items():
            if not hasattr(self, key):
                raise ValueError("Unknown key {} in {} object".format(key, type(self).__name__))

            setattr(self, key, value)

    def connect(self, task):
        """
        Connect to a task.

        When running locally, the task will save the parameters from self.
        When running with a worker, self will be updated according to the task's
        saved parameters.

        :param task: The task to connect to.
        :type task: .Task
        """

        return task.connect(self)


# clearml/version.py

__version__ = '1.6.4'


# clearml/utilities/__init__.py

from .dicts import Logs

__all__ = ["Logs"]


# clearml/utilities/async_manager.py

import os
import time
import six

from .process.mp import SingletonLock


class AsyncManagerMixin(object):
    _async_results_lock = SingletonLock()
    # per pid (process) list of async jobs (support for sub-processes forking)
    _async_results = {}

    @classmethod
    def _add_async_result(cls, result, wait_on_max_results=None, wait_time=30, wait_cb=None):
        while True:
            try:
                cls._async_results_lock.acquire()
                # discard completed results
                pid = os.getpid()
                cls._async_results[pid] = [r for r in cls._async_results.get(pid, []) if not r.ready()]
                num_results = len(cls._async_results[pid])
                if wait_on_max_results is not None and num_results >= wait_on_max_results:
                    # At least max_results results are still pending, wait
                    if wait_cb:
                        wait_cb(num_results)
                    if wait_time:
                        time.sleep(wait_time)
                    continue
                # add result
                if result and not result.ready():
                    if not cls._async_results.get(pid):
                        cls._async_results[pid] = []
                    cls._async_results[pid].append(result)
                break
            finally:
                cls._async_results_lock.release()

    @classmethod
    def wait_for_results(cls, timeout=None, max_num_uploads=None):
        remaining = timeout
        count = 0
        pid = os.getpid()
        for r in cls._async_results.get(pid, []):
            if r.ready():
                continue
            t = time.time()
            # bugfix for python2.7 threading issues
            if six.PY2 and not remaining:
                while not r.ready():
                    r.wait(timeout=2.)
            else:
                r.wait(timeout=remaining)
            count += 1
            if max_num_uploads is not None and max_num_uploads - count <= 0:
                break
            if timeout is not None:
                remaining = max(0., remaining - max(0., time.time() - t))
                if not remaining:
                    break

    @classmethod
    def get_num_results(cls):
        pid = os.getpid()
        if cls._async_results.get(pid, []):
            return len([r for r in cls._async_results.get(pid, []) if not r.ready()])
        else:
            return 0


# clearml/utilities/attrs.py

import attr

from .version import Version


class attrs(object):
    def __init__(self, *args, **kwargs):
        if any(x in kwargs for x in ("eq", "order")):
            raise RuntimeError("Only `cmp` is supported for attr.attrs, not `eq` or `order`")
        if Version(attr.__version__) >= Version("19.2"):
            cmp = kwargs.pop("cmp", None)
            if cmp is not None:
                kwargs["eq"] = kwargs["order"] = cmp
        self.args = args
        self.kwargs = kwargs

    def __call__(self, f):
        return attr.attrs(*self.args, **self.kwargs)(f)


# clearml/utilities/check_updates.py

from __future__ import absolute_import, division, print_function

import json
import os

import requests

from ..backend_api.session import Session
from ..backend_config import EnvEntry

from .version import Version


class CheckPackageUpdates(object):
    _package_version_checked = False

    @classmethod
    def check_new_package_available(cls, only_once=False):
        """
        :return: True, if there is a newer package in PyPI.
        """
        if only_once and cls._package_version_checked:
            return None

        # noinspection PyBroadException
        try:
            cls._package_version_checked = True
            client, version = Session._client[0]
            version = Version(version)
            is_demo = 'https://demoapi.demo.clear.ml/'.startswith(Session.get_api_server_host())

            update_server_releases = requests.get(
                'https://updates.clear.ml/updates',
                json={"demo": is_demo,
                      "versions": {c: str(v) for c, v in Session._client},
                      "CI": str(os.environ.get('CI', ''))},
                timeout=3.0
            )

            if update_server_releases.ok:
                update_server_releases = update_server_releases.json()
            else:
                return None

            client_answer = update_server_releases.get(client, {})
            if "version" not in client_answer:
                return None

            # do not output upgrade message if we are running inside a CI process.
            if EnvEntry("CI", type=bool, ignore_errors=True).get():
                return None

            latest_version = Version(client_answer["version"])

            if version >= latest_version:
                return None
            not_patch_upgrade = latest_version.release[:2] == version.release[:2]
            return str(latest_version), not_patch_upgrade, client_answer.get("description").split("\r\n")
        except Exception:
            return None

    @staticmethod
    def get_version_from_updates_server(cur_version):
        """
        Get the latest version for clearml from updates server
        :param cur_version: The current running version of clearml
        :type cur_version: Version
        """
        try:
            _ = requests.get('https://updates.clear.ml/updates',
                             data=json.dumps({"versions": {"clearml": str(cur_version)}}),
                             timeout=1.0)
            return
        except Exception:
            pass


# clearml/utilities/config.py

from __future__ import division

import json

import six
import pyparsing

from .dicts import hocon_quote_key, hocon_unquote_key
from .pyhocon import ConfigFactory, HOCONConverter
from ..storage.util import parse_size


def parse_human_size(value):
    if isinstance(value, six.string_types):
        return parse_size(value)
    return value


def get_percentage(config, key, required=True, default=None):
    if required:
        value = config.get(key)
    else:
        value = config.get(key, default)
        if value is None:
            return
    try:
        if isinstance(value, six.string_types):
            value = value.strip()
            if value.endswith('%'):
                # "50%" => 0.5
                return float(value.strip('%')) / 100.
            # "50" => 50

        value = float(value)
        if value < 1:
            # 0.5 => 50% => 0.5
            return value

        # 50 => 0.5, 10.5 => 0.105
        return value / 100.

    except ValueError as e:
        raise ValueError('Config: failed parsing %s: %s' % (key, e))


def get_human_size_default(config, key, default=None):
    raw_value = config.get(key, default)

    if raw_value is None:
        return default

    return parse_human_size(raw_value)


def config_dict_to_text(config):
    # if already string return as is
    if isinstance(config, six.string_types):
        return config
    if not isinstance(config, (dict, list)):
        raise ValueError("Configuration only supports dictionary/list objects")
    try:
        # noinspection PyBroadException
        try:
            text = HOCONConverter.to_hocon(ConfigFactory.from_dict(hocon_quote_key(config)))
        except Exception:
            # fallback json+pyhocon
            # hack, pyhocon is not very good with dict conversion so we pass through json
            import json
            text = json.dumps(config)
            text = HOCONConverter.to_hocon(ConfigFactory.parse_string(text))

    except Exception:
        raise ValueError("Could not serialize configuration dictionary:\n", config)
    return text


def text_to_config_dict(text):
    if not isinstance(text, six.string_types):
        raise ValueError("Configuration parsing only supports string")
    # noinspection PyBroadException
    try:
        return hocon_unquote_key(ConfigFactory.parse_string(text))
    except pyparsing.ParseBaseException as ex:
        pos = "at char {}, line:{}, col:{}".format(ex.loc, ex.lineno, ex.column)
        six.raise_from(ValueError("Could not parse configuration text ({}):\n{}".format(pos, text)), None)
    except Exception:
        six.raise_from(ValueError("Could not parse configuration text:\n{}".format(text)), None)


def verify_basic_value(value):
    # return True if value of of basic type (json serializable)
    if not isinstance(value,
                      six.string_types + six.integer_types +
                      (six.text_type, float, list, tuple, dict, type(None))):
        return False
    try:
        json.dumps(value)
        return True
    except TypeError:
        return False


# clearml/utilities/deferred.py

import threading
from functools import wraps

import attr
import six


class DeferredExecutionPool(object):
    @attr.s
    class _DeferredAction(object):
        method = attr.ib()
        args = attr.ib()
        kwargs = attr.ib()

    def __init__(self, instance):
        self._instance = instance
        self._pool = []
        self._lock = threading.Lock()

    def add(self, callable_, *args, **kwargs):
        self._pool.append(self._DeferredAction(callable_, args, kwargs))

    def clear(self):
        with self._lock:
            pool = self._pool
            self._pool = []
            return pool

    def apply(self):
        pool = self.clear()
        for action in pool:
            action.method(self._instance, *action.args, **action.kwargs)

    def copy_from(self, other):
        if not isinstance(self._instance, type(other._instance)):
            raise ValueError("Copy deferred actions must be with the same instance type")

        self._pool = other._pool[:]


class ParameterizedDefaultDict(dict):
    def __init__(self, factory, *args, **kwargs):
        super(ParameterizedDefaultDict, self).__init__(*args, **kwargs)
        self._factory = factory

    def __missing__(self, key):
        self[key] = self._factory(key)
        return self[key]


class DeferredExecution(object):
    def __init__(self, pool_cls=DeferredExecutionPool):
        self._pools = ParameterizedDefaultDict(pool_cls)

    def __get__(self, instance, owner):
        if not instance:
            return self

        return self._pools[instance]

    def defer_execution(self, condition_or_attr_name=True):
        """
        Deferred execution decorator, designed to wrap class functions for classes containing a deferred execution pool.
        :param condition_or_attr_name: Condition controlling whether wrapped function should be deferred.
            True by default. If a callable is provided, it will be called with the class instance (self)
            as first argument. If a string is provided, a class instance (self) attribute by that name is evaluated.
        :return:
        """
        def decorator(func):
            @wraps(func)
            def wrapper(instance, *args, **kwargs):
                if self._resolve_condition(instance, condition_or_attr_name):
                    self._pools[instance].add(func, *args, **kwargs)
                else:
                    return func(instance, *args, **kwargs)
            return wrapper
        return decorator

    @staticmethod
    def _resolve_condition(instance, condition_or_attr_name):
        if callable(condition_or_attr_name):
            return condition_or_attr_name(instance)
        elif isinstance(condition_or_attr_name, six.string_types):
            return getattr(instance, condition_or_attr_name)
        return condition_or_attr_name

    def _apply(self, instance, condition_or_attr_name):
        if self._resolve_condition(instance, condition_or_attr_name):
            self._pools[instance].apply()

    def apply_after(self, condition_or_attr_name=True):
        """
        Decorator for applying deferred execution pool after wrapped function has completed
        :param condition_or_attr_name: Condition controlling whether deferred pool should be applied. True by default.
            If a callable is provided, it will be called with the class instance (self) as first argument.
            If a string is provided, a class instance (self) attribute by that name is evaluated.
        """
        def decorator(func):
            @wraps(func)
            def wrapper(instance, *args, **kwargs):
                res = func(instance, *args, **kwargs)
                self._apply(instance, condition_or_attr_name)
                return res
            return wrapper
        return decorator

    def apply_before(self, condition_or_attr_name=True):
        """
        Decorator for applying deferred execution pool before wrapped function is executed
        :param condition_or_attr_name: Condition controlling whether deferred pool should be applied. True by default.
            If a callable is provided, it will be called with the class instance (self) as first argument.
            If a string is provided, a class instance (self) attribute by that name is evaluated.
        """
        def decorator(func):
            @wraps(func)
            def wrapper(instance, *args, **kwargs):
                self._apply(instance, condition_or_attr_name)
                return func(instance, *args, **kwargs)
            return wrapper
        return decorator


# clearml/utilities/dicts.py

""" Utilities """
from typing import Optional, Any

_epsilon = 0.00001


class ReadOnlyDict(dict):
    def __readonly__(self, *args, **kwargs):
        raise ValueError("This is a read only dictionary")
    __setitem__ = __readonly__
    __delitem__ = __readonly__
    pop = __readonly__
    popitem = __readonly__
    clear = __readonly__
    update = __readonly__
    setdefault = __readonly__
    del __readonly__


class Logs:
    _logs_instances = []

    def __init__(self, data=None):
        self._data = data or {}
        self._logs_instances.append(self)

    def reset(self):
        self._data = {}

    @property
    def data(self):
        return self._data

    @classmethod
    def get_instances(cls):
        return cls._logs_instances


class BlobsDict(dict):
    """
    Overloading getitem so that the 'data' copy is only done when the dictionary item is accessed.
    """

    def __init__(self, *args, **kwargs):
        super(BlobsDict, self).__init__(*args, **kwargs)

    def __getitem__(self, k):
        val = super(BlobsDict, self).__getitem__(k)
        if isinstance(val, dict):
            return BlobsDict(val)
        # We need to ask isinstance without actually importing blob here
        # so we accept that in order to appreciate beauty in life we must have a dash of ugliness.
        # ans instead of -
        # elif isinstance(val, Blob):
        # we ask:
        elif hasattr(val, '__class__') and val.__class__.__name__ == 'Blob':
            return val.data
        else:
            return val


class NestedBlobsDict(BlobsDict):
    """A dictionary that applies an arbitrary key-altering function
       before accessing the keys."""

    def __init__(self, *args, **kwargs):
        super(NestedBlobsDict, self).__init__(*args, **kwargs)

    def __getitem__(self, keys_str=''):

        if keys_str == '':
            return super(NestedBlobsDict, self).__getitem__(self)

        keylist = keys_str.split('.')

        cur = super(NestedBlobsDict, self).__getitem__(keylist[0])
        if len(keylist) == 1:
            return cur
        else:
            return NestedBlobsDict(cur)['.'.join(keylist[1:])]

    def __contains__(self, keys_str):
        keylist = self.keys()
        return keys_str in keylist

    def as_dict(self):
        return dict(self)

    def get(self, keys_str, default=None):
        # noinspection PyBroadException
        try:
            return self[keys_str]
        except Exception:
            return None

    def _keys(self, cur_dict, path):
        deep_keys = []
        cur_keys = dict.keys(cur_dict)

        for key in cur_keys:
            if isinstance(cur_dict[key], dict):
                if len(path) > 0:
                    deep_keys.extend(self._keys(cur_dict[key], path + '.' + key))
                else:
                    deep_keys.extend(self._keys(cur_dict[key], key))
            else:
                if len(path) > 0:
                    deep_keys.append(path + '.' + key)
                else:
                    deep_keys.append(key)

        return deep_keys

    def keys(self):
        return self._keys(self, '')


def merge_dicts(dict1, dict2):
    """ Recursively merges dict2 into dict1 """
    if not isinstance(dict1, dict) or not isinstance(dict2, dict):
        return dict2
    for k in dict2:
        if k in dict1:
            dict1[k] = merge_dicts(dict1[k], dict2[k])
        else:
            dict1[k] = dict2[k]
    return dict1


def hocon_quote_key(a_obj):
    """ Recursively quote key with '.' to \"key\" """
    if isinstance(a_obj, list):
        return [hocon_quote_key(a) for a in a_obj]
    elif isinstance(a_obj, tuple):
        return tuple(hocon_quote_key(a) for a in a_obj)
    elif not isinstance(a_obj, dict):
        return a_obj

    # preserve dict type
    a_dict = a_obj
    new_dict = type(a_dict)()
    for k, v in a_dict.items():
        if isinstance(k, str) and '.' in k:
            new_dict['"{}"'.format(k)] = hocon_quote_key(v)
        else:
            new_dict[k] = hocon_quote_key(v)
    return new_dict


def hocon_unquote_key(a_obj):
    """ Recursively unquote \"key\" with '.' to key """

    if isinstance(a_obj, list):
        return [hocon_unquote_key(a) for a in a_obj]
    elif isinstance(a_obj, tuple):
        return tuple(hocon_unquote_key(a) for a in a_obj)
    elif not isinstance(a_obj, dict):
        return a_obj

    a_dict = a_obj

    # ConfigTree to dict
    if hasattr(a_dict, 'as_plain_ordered_dict'):
        a_dict = a_dict.as_plain_ordered_dict()

    # preserve dict type
    new_dict = type(a_dict)()
    for k, v in a_dict.items():
        if isinstance(k, str) and k[0] == '"' and k[-1] == '"' and '.' in k:
            new_dict[k[1:-1]] = hocon_unquote_key(v)
        else:
            new_dict[k] = hocon_unquote_key(v)
    return new_dict


def cast_str_to_bool(value, strip=True):
    # type: (Any, bool) -> Optional[bool]
    a_strip_v = value if not strip else str(value).lower().strip()
    if a_strip_v == 'false' or not a_strip_v:
        return False
    elif a_strip_v == 'true':
        return True
    else:
        # first try to cast to integer
        try:
            return bool(int(a_strip_v))
        except (ValueError, TypeError):
            return None


# clearml/utilities/enum.py

""" Enum utilities """


class EnumOptions(object):
    """ Base class for enum-like classes using class-attributes with string values to represent enum key/value pairs """
    __cache = None

    @classmethod
    def values(cls):
        """ Extract list of enum-like options based on the derived classes' attributes.
            Any class attribute who's key doesn't start with an underscore and who's value is not a class method
             or callable is considered an option.
            Returns a list of attribute names representing the options.
        """
        if cls.__cache is None:
            cls.__cache = [v for k, v in vars(cls).items() if
                           not k.startswith('_') and not callable(v) and not isinstance(v, classmethod)]
        return cls.__cache


class Options(object):
    @classmethod
    def _all(cls):
        return {k: v for k, v in vars(cls) if not k.startswith('_')}


# clearml/utilities/files.py

import os
from sys import platform

import pathlib2
import psutil


def get_filename_max_length(dir_path):
    # type: (str) -> int
    try:
        dir_path = pathlib2.Path(os.path.abspath(dir_path))
        if platform == "win32":
            dir_drive = dir_path.drive
            for drv in psutil.disk_partitions():
                if drv.device.startswith(dir_drive):
                    return drv.maxfile
        elif platform in ("linux", "darwin"):
            return os.statvfs(dir_path).f_namemax
    except Exception as err:
        print(err)

    return 255  # Common filesystems like NTFS, EXT4 and HFS+ limited with 255


# clearml/utilities/io_manager.py

class IOCallsManager(object):

    def __init__(self):
        self.threads_io = {}

    def add_io_to_thread(self, thread_id, io_object):
        if thread_id in self.threads_io:
            self.threads_io[thread_id].add(id(io_object))
        else:
            self.threads_io[thread_id] = {id(io_object)}
        if self._io_has_canvas_figure(io_object):
            self.threads_io[thread_id].add(id(io_object.canvas.figure))

    def is_plot_called(self, thread_id, io_object):
        return id(io_object) in self.threads_io.get(thread_id, set())

    def remove_io_to_thread(self, thread_id, io_object):
        try:
            self.threads_io[thread_id].remove(id(io_object))
            if self._io_has_canvas_figure(io_object):
                self.threads_io[thread_id].remove(id(io_object.canvas.figure))
        except Exception:
            pass

    def remove_thread(self, thread_id):
        if thread_id in self.threads_io:
            del self.threads_io[thread_id]

    @staticmethod
    def _io_has_canvas_figure(io_object):
        return hasattr(io_object, 'canvas') and hasattr(io_object.canvas, 'figure')


# clearml/utilities/matching.py

from pathlib2 import Path
from fnmatch import fnmatch
from typing import Union


def matches_any_wildcard(path, wildcards, recursive=True):
    # type: (str, Union[str, list], bool) -> bool
    """
    Checks if given pattern matches any supplied wildcard

    :param path: path to check
    :param wildcards: wildcards to check against
    :param recursive: whether or not the check is recursive. Default: True
        E.g. for path='directory/file.ext' and wildcards='*.ext',
        recursive=False will return False, but recursive=True will
        return True

    :return: True if the path matches any wildcard and False otherwise
    """
    path = Path(path).as_posix()
    if wildcards is None:
        wildcards = ["*"]
    if not isinstance(wildcards, list):
        wildcards = [wildcards]
    wildcards = [str(w) for w in wildcards]
    if not recursive:
        path = path.split("/")
    for wildcard in wildcards:
        if not recursive:
            wildcard = wildcard.split("/")
            matched = True
            if len(path) != len(wildcard):
                continue
            for path_segment, wildcard_segment in zip(path, wildcard):
                if not fnmatch(path_segment, wildcard_segment):
                    matched = False
                    break
            if matched:
                return True
        else:
            wildcard_file = wildcard.split("/")[-1]
            wildcard_dir = wildcard[: -len(wildcard_file)] + "*"
            if fnmatch(path, wildcard_dir) and fnmatch("/" + path, "*/" + wildcard_file):
                return True


# clearml/utilities/parallel.py

import os

from copy import deepcopy
from time import sleep
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Callable, Optional, Union, Generator, List, Type
from zipfile import ZipFile, ZIP_DEFLATED
from six.moves.queue import PriorityQueue, Queue, Empty
from pathlib2 import Path
from tempfile import mkstemp
from collections import deque
from threading import Thread

from ..debugging.log import LoggerRoot
from ..storage.util import format_size


class _DeferredClass(object):
    __slots__ = ('__queue', '__future_caller', '__future_func')

    def __init__(self, a_future_caller, future_func):
        self.__queue = Queue()
        self.__future_caller = a_future_caller
        self.__future_func = future_func

    def __nested_caller(self, item, args, kwargs):
        # wait until object is constructed
        getattr(self.__future_caller, "id")  # noqa

        future_func = getattr(self.__future_caller, self.__future_func)
        the_object = future_func()
        the_object_func = getattr(the_object, item)
        return the_object_func(*args, **kwargs)

    def _flush_into_logger(self, a_future_object=None, a_future_func=None):
        self.__close_queue(a_future_object=a_future_object, a_future_func=a_future_func)

    def __close_queue(self, a_future_object=None, a_future_func=None):
        # call this function when we Know the object is initialization is completed
        if self.__queue is None:
            return

        _queue = self.__queue
        self.__queue = None
        while True:
            # noinspection PyBroadException
            try:
                item, args, kwargs = _queue.get(block=False)
                if a_future_object:
                    future_func = getattr(a_future_object, self.__future_func)
                    the_object = future_func()
                    the_object_func = getattr(the_object, item)
                    the_object_func(*args, **kwargs)
                elif a_future_func:
                    the_object_func = getattr(a_future_func, item)
                    the_object_func(*args, **kwargs)
                else:
                    self.__nested_caller(item, args, kwargs)
            except Empty:
                break
            except Exception:
                # import traceback
                # stdout_print(''.join(traceback.format_exc()))
                pass

    def __getattr__(self, item):
        def _caller(*args, **kwargs):
            # if we already completed the background initialization, call functions immediately
            # noinspection PyProtectedMember
            if not self.__queue or self.__future_caller._FutureTaskCaller__executor is None:
                return self.__nested_caller(item, args, kwargs)

            # noinspection PyBroadException
            try:
                # if pool is still active call async
                self.__queue.put((item, deepcopy(args) if args else args, deepcopy(kwargs) if kwargs else kwargs))
            except Exception:
                # assume we wait only if self.__pool was nulled between the if and now, so just call directly
                return self.__nested_caller(item, args, kwargs)

            # let's hope it is the right one
            return True

        return _caller


class FutureTaskCaller(object):
    """
    FutureTaskCaller is used to create a class via a functions async, in another thread.

    For example:

    .. code-block:: py

        future = FutureTaskCaller().call(func=max, func_cb=None, override_cls=None, 1, 2)
        print('Running other code')
        print(future.result())  # will print '2'
    """
    __slots__ = ('__object', '__object_cls', '__executor', '__deferred_bkg_class')

    @property
    def __class__(self):
        return self.__object_cls

    def __init__(self, func, func_cb, override_cls, *args, **kwargs):
        # type: (Callable, Optional[Callable], Type, *Any, **Any) -> None
        """
        __init__(*args, **kwargs) in another thread

        :return: This FutureTaskCaller instance
        """
        self.__object = None
        self.__object_cls = override_cls
        self.__deferred_bkg_class = _DeferredClass(self, "get_logger")

        self.__executor = Thread(target=self.__submit__, args=(func, func_cb, args, kwargs))
        self.__executor.daemon = True
        self.__executor.start()

    def __submit__(self, fn, fn_cb, args, kwargs):
        # background initialization call
        _object = fn(*args, **kwargs)

        # push all background calls (now that the initialization is complete)
        if self.__deferred_bkg_class:
            _deferred_bkg_class = self.__deferred_bkg_class
            self.__deferred_bkg_class = None
            # noinspection PyProtectedMember
            _deferred_bkg_class._flush_into_logger(a_future_object=_object)

        # store the initialized object
        self.__object = _object
        # callback function
        if fn_cb is not None:
            fn_cb(self.__object)

    def __getattr__(self, item):
        # if we get here, by definition this is not a __slot__ entry, pass to the object
        return getattr(self.__result__(), item)

    def __setattr__(self, item, value):
        # make sure we can set the slots
        if item in ["_FutureTaskCaller__executor", "_FutureTaskCaller__object",
                    "_FutureTaskCaller__object_cls", "_FutureTaskCaller__deferred_bkg_class"]:
            return super(FutureTaskCaller, self).__setattr__(item, value)

        setattr(self.__result__(), item, value)

    def __result__(self, timeout=None):
        # type: (Optional[float]) -> Any
        """
        Wait and get the result of the function called with self.call()

        :param timeout: The maximum number of seconds to wait for the result. If None,
            there is no limit for the wait time.

        :return: The result of the called function
        """
        if self.__executor:
            # since the test is not atomic, we assume that if we failed joining
            # it is because someone else joined before us
            # noinspection PyBroadException
            try:
                self.__executor.join(timeout=timeout)
            except RuntimeError:
                # this is probably calling ourselves from the same thread
                raise
            except Exception:
                # wait until that someone else updated the __object
                while self.__object is None:
                    sleep(1)
            self.__executor = None
        return self.__object

    # This is the part where we are no longer generic, but __slots__
    # inheritance is too cumbersome to actually inherit and make sure it works optimally
    def get_logger(self):
        if self.__object is not None:
            return self.__object.get_logger()

        if self.__deferred_bkg_class is None:
            # we are shutting down, wait until object is available
            return self.__result__().get_logger()

        return self.__deferred_bkg_class


class ParallelZipper(object):
    """
    Used to zip multiple files in zip chunks of a particular size, all in parallel
    """
    class ZipperObject(object):
        def __init__(
            self,
            chunk_size,  # int
            zipper_queue,  # PriorityQueue[ParallelZipper.ZipperObject]
            zipper_results,  # Queue[ParallelZipper.ZipperObject]
            allow_zip_64,  # bool
            compression,  # Any
            zip_prefix,  # str
            zip_suffix,  # str
            verbose,  # bool
            task,  # Any
        ):
            # (...) -> ParallelZipper.ZipperObject
            """
            Initialize a ParallelZipper.ZipperObject instance that holds its corresponding zip
            file, as well as other relevant data

            :param chunk_size: Chunk size, in MB. The ParallelZipper will try its best not to exceed this size
                when bulding up this zipper object, but that is not guaranteed
            :param zipper_queue: PriorityQueue that holds ParallelZipper.ZipperObject instances.
                When this ParallelZipper.ZipperObject can hold more data (i.e. chunk_size was not exceeded),
                this object will reinsert itself in this queue to be reused by the ParallelZipper.
                Else, a fresh ParallelZipper.ZipperObject will be inserted
            :param zipper_results: Queue that holds ParallelZipper.ZipperObject instances. These instances
                are added to this queue when chunk_size is exceeded
            :param allow_zip_64: if True ZipFile will create files with ZIP64 extensions when
                needed, otherwise it will raise an exception when this would be necessary
            :param compression: ZipFile.ZIP_STORED (no compression), ZipFile.ZIP_DEFLATED (requires zlib),
                ZipFile.ZIP_BZIP2 (requires bz2) or ZipFile.ZIP_LZMA (requires lzma).
            :param zip_prefix: The zip file created by this object will have its name prefixed by this
            :param zip_suffix: The zip file created by this object will have its name suffixed by this
            :param verbose: If True, print data relevant to the file compression
            :param task: ClearML Task, used for logging

            :return: ParallelZipper.ZipperObject instance
            """
            self._chunk_size = chunk_size
            self._zipper_queue = zipper_queue
            self._zipper_results = zipper_results
            self._allow_zip_64 = allow_zip_64
            self._compression = compression
            self._zip_prefix = zip_prefix
            self._zip_suffix = zip_suffix
            self._verbose = verbose
            self._task = task
            self.fd, self.zip_path = mkstemp(prefix=zip_prefix, suffix=zip_suffix)
            self.zip_path = Path(self.zip_path)
            self.zip_file = ZipFile(self.zip_path.as_posix(), "w", allowZip64=allow_zip_64, compression=compression)
            self.archive_preview = ""
            self.count = 0
            self.files_zipped = set()

        def zip(self, file_path, arcname=None):
            # type: (Union[str, Path], str) -> ()
            """
            Zips a file into the ZipFile created by this instance. This instance will either add
            itself back to the PriorityQueue used to select the best zipping candidate or add itself
            to the result Queue after exceeding self.chunk_size.

            :param file_path: Path to the file to be zipped
            :param arcname: Name of the file in the archive
            """
            if self._verbose and self._task:
                self._task.get_logger().report_text("Compressing {}".format(Path(file_path).as_posix()))
            self.zip_file.write(file_path, arcname=arcname)
            self.count += 1
            preview_path = arcname
            if not preview_path:
                preview_path = file_path
            self.archive_preview += "{} - {}\n".format(preview_path, format_size(self.size))
            self.files_zipped.add(Path(file_path).as_posix())
            if self._chunk_size <= 0 or self.size < self._chunk_size:
                self._zipper_queue.put(self)
            else:
                self._zipper_queue.put(
                    ParallelZipper.ZipperObject(
                        self._chunk_size,
                        self._zipper_queue,
                        self._zipper_results,
                        self._allow_zip_64,
                        self._compression,
                        self._zip_prefix,
                        self._zip_suffix,
                        self._verbose,
                        self._task,
                    )
                )
                self._zipper_results.put(self)

        def merge(self, other):
            # type: (ParallelZipper.ZipperObject) -> ()
            """
            Merges one ParallelZipper.ZipperObject instance into the current one.
            All the files zipped by the other instance will be added to this instance,
            as well as any other useful additional data

            :param other: ParallelZipper.ZipperObject instance to merge into this one
            """
            with ZipFile(self.zip_path.as_posix(), "a") as parent_zip:
                with ZipFile(other.zip_path.as_posix(), "r") as child_zip:
                    for child_name in child_zip.namelist():
                        parent_zip.writestr(child_name, child_zip.open(child_name).read())
            self.files_zipped |= other.files_zipped
            self.count += other.count
            self.archive_preview += other.archive_preview

        def close(self):
            # type: () -> ()
            """
            Attempts to close file descriptors associated to the ZipFile
            """
            # noinspection PyBroadException
            try:
                self.zip_file.close()
                os.close(self.fd)
            except Exception:
                pass

        def delete(self):
            # type: () -> ()
            """
            Attempts to delete the ZipFile from the disk
            """
            # noinspection PyBroadException
            try:
                self.close()
                self.zip_path.unlink()
            except Exception:
                pass

        @property
        def size(self):
            # type: () -> ()
            """
            :return: Size of the ZipFile, in bytes
            """
            return self.zip_path.stat().st_size

        def __lt__(self, other):
            # we want to completely "fill" as many zip files as possible, hence the ">" comparison
            return self.size > other.size

    def __init__(
        self,
        chunk_size,  # type: int
        max_workers,  # type: int
        allow_zip_64=True,  # type: Optional[bool]
        compression=ZIP_DEFLATED,  # type: Optional[Any]
        zip_prefix="",  # type: Optional[str]
        zip_suffix="",  # type: Optional[str]
        verbose=False,  # type: Optional[bool]
        task=None,  # type: Optional[Any]
        pool=None,  # type: Optional[ThreadPoolExecutor]
    ):
        # type: (...) -> ParallelZipper
        """
        Initialize the ParallelZipper. Each zip created by this object will have the following naming
        format: [zip_prefix]<random_string>[zip_suffix]

        :param chunk_size: Chunk size, in MB. The ParallelZipper will try its best not to exceed this size,
            but that is not guaranteed
        :param max_workers: The maximum number of workers spawned when zipping the files
        :param allow_zip_64: if True ZipFile will create files with ZIP64 extensions when
            needed, otherwise it will raise an exception when this would be necessary
        :param compression: ZipFile.ZIP_STORED (no compression), ZipFile.ZIP_DEFLATED (requires zlib),
            ZipFile.ZIP_BZIP2 (requires bz2) or ZipFile.ZIP_LZMA (requires lzma).
        :param zip_prefix: Zip file names will be prefixed by this
        :param zip_suffix: Zip file names will pe suffixed by this
        :param verbose: If True, print data relevant to the file compression
        :param task: ClearML Task, used for logging
        :param pool: Use this ThreadPoolExecutor instead of creating one. Note that this pool will not be
            closed after zipping is finished.

        :return: ParallelZipper instance
        """
        self._chunk_size = chunk_size * (1024 ** 2)
        self._max_workers = max_workers
        self._allow_zip_64 = allow_zip_64
        self._compression = compression
        self._zip_prefix = zip_prefix
        self._zip_suffix = zip_suffix
        self._verbose = verbose
        self._task = task
        self._pool = pool
        self._zipper_queue = PriorityQueue()
        self._zipper_results = Queue()

    def zip_iter(self, file_paths, arcnames={}):
        # type: (List[Union(str, Path)], Optional[dict[Union(str, Path), str]]) -> Generator[ParallelZipper.ZipperObject]
        """
        Generator function that returns zip files as soon as they are available.
        The zipping is done in parallel

        :param file_paths: List of paths to the files to zip
        :param arcnames: Dictionary that maps the file path to what should be its name in the archive.

        :return: Generator of ParallelZipper.ZipperObjects
        """
        while not self._zipper_queue.empty():
            self._zipper_queue.get_nowait()
        for _ in range(self._max_workers):
            self._zipper_queue.put(
                ParallelZipper.ZipperObject(
                    self._chunk_size,
                    self._zipper_queue,
                    self._zipper_results,
                    self._allow_zip_64,
                    self._compression,
                    self._zip_prefix,
                    self._zip_suffix,
                    self._verbose,
                    self._task,
                )
            )
        filtered_file_paths = []
        for file_path in file_paths:
            if not Path(file_path).is_file():
                LoggerRoot.get_base_logger().warning("Could not store dataset file {}. File skipped".format(file_path))
            else:
                filtered_file_paths.append(file_path)
        file_paths = filtered_file_paths

        file_paths = sorted(file_paths, key=lambda k: Path(k).stat().st_size, reverse=True)
        # zip in parallel
        pooled = []
        if not self._pool:
            pool = ThreadPoolExecutor(max_workers=self._max_workers)
        else:
            pool = self._pool
        for f in file_paths:
            zipper = self._zipper_queue.get()
            pooled.append(pool.submit(zipper.zip, Path(f).as_posix(), arcname=arcnames.get(f)))
            for result in self._yield_zipper_results():
                yield result
        for task in pooled:
            task.result()
        if not self._pool:
            pool.close()

        for result in self._yield_zipper_results():
            yield result

        zipper_results_leftover = []

        # extract remaining results
        while not self._zipper_queue.empty():
            result = self._zipper_queue.get()
            if result.count != 0:
                zipper_results_leftover.append(result)
            else:
                result.delete()
        zipper_results_leftover = deque(sorted(zipper_results_leftover, reverse=True))

        # merge zip files greedily if possible and get the paths as results
        while len(zipper_results_leftover) > 0:
            zip_ = zipper_results_leftover.pop()
            zip_.close()
            if zip_.size >= self._chunk_size > 0:
                yield zip_
                continue
            while len(zipper_results_leftover) > 0 and (
                self._chunk_size <= 0 or zipper_results_leftover[0].size + zip_.size < self._chunk_size
            ):
                child_zip = zipper_results_leftover.popleft()
                child_zip.close()
                zip_.merge(child_zip)
                child_zip.delete()
            yield zip_

    def _yield_zipper_results(self):
        while True:
            try:
                result = self._zipper_results.get_nowait()
                result.close()
                yield result
            except Empty:
                break


# clearml/utilities/plotly_reporter.py

import json
import numpy as np

from ..errors import UsageError
from ..utilities.dicts import merge_dicts

try:
    import pandas as pd
except ImportError:
    pd = None
from attr import attrs, attrib


def create_2d_histogram_plot(np_row_wise, labels, title=None, xtitle=None, ytitle=None, series=None, xlabels=None,
                             comment=None, mode='group', layout_config=None):
    """
    Create a 2D Plotly histogram chart from a 2D numpy array
    :param np_row_wise: 2D numpy data array
    :param labels: Histogram labels
    :param title: Chart title
    :param xtitle: X-Series title
    :param ytitle: Y-Series title
    :param comment: comment underneath the title
    :param mode: multiple histograms mode. valid options are: stack / group / relative. Default is 'group'.
    :param layout_config: optional extra layout configuration
    :return: Plotly chart dict.
    """
    assert mode in ('stack', 'group', 'relative')

    np_row_wise = np.atleast_2d(np_row_wise)
    assert len(np_row_wise.shape) == 2, "Expected a 2D numpy array"
    use_series = bool(labels)

    # using labels without xlabels leads to original behavior
    if labels is not None and xlabels is None:
        assert len(labels) == np_row_wise.shape[0], "Please provide a label for each data row"
    elif xlabels is None:
        fake_label = series or ''
        labels = [fake_label] * np_row_wise.shape[0]
    elif labels:
        if len(labels) == 1:
            labels = [labels] * np_row_wise.shape[0]
        assert len(xlabels) == np_row_wise.shape[1]
    elif not labels and xlabels:
        labels = [series]

    data = [_np_row_to_plotly_data_item(np_row=np_row_wise[i, :], label=labels[i] if labels else None, xlabels=xlabels)
            for i in range(np_row_wise.shape[0])]
    return _plotly_hist_dict(title=series if use_series else title,
                             xtitle=xtitle, ytitle=ytitle, mode=mode, data=data, comment=comment,
                             layout_config=layout_config)


def _to_np_array(value):
    if not isinstance(value, np.ndarray):
        value = np.array(value)

    return value


@attrs
class SeriesInfo(object):
    name = attrib(type=str)
    data = attrib(type=np.ndarray, converter=_to_np_array)
    labels = attrib(default=None)

    @data.validator
    def _validate_data(self, _, value):
        if value.ndim != 2:
            raise ValueError("Expected series data to be 2D numpy array")

        if value.shape[1] != 2:
            raise ValueError("Expected series data to have 2 columns")

    def __attrs_post_init__(self):
        if (self.labels is not None) and (len(self.labels) != self.data.shape[0]):
            raise ValueError(
                "If 'labels' is provided, it must be a list or tuple, "
                "the same length as the data"
            )


def create_line_plot(title, series, xtitle, ytitle, mode='lines', reverse_xaxis=False,
                     comment=None, MAX_SIZE=None, layout_config=None):
    plotly_obj = _plotly_scatter_layout_dict(
        title=title if not comment else (title + '<br><sup>' + comment + '</sup>'),
        xaxis_title=xtitle,
        yaxis_title=ytitle,
    )

    if reverse_xaxis:
        plotly_obj["layout"]["xaxis"]["autorange"] = "reversed"

    # check maximum size of data
    MAX_SIZE = MAX_SIZE or 800000
    series_sizes = [s.data.size for s in series]
    total_size = sum(series_sizes)
    if total_size > MAX_SIZE:
        # we need to downscale
        base_size = MAX_SIZE / len(series_sizes)
        baseused_size = sum([min(s, base_size) for s in series_sizes])
        leftover = MAX_SIZE - baseused_size
        for s in series:
            # if we need to down-sample, use low-pass average filter and sampling
            if s.data.size >= base_size:
                budget = base_size
                # if we have some leftover in the budget, split based on series sizes
                if leftover > 0:
                    # calculate the relative overflow  of this series compared to all the overflows
                    # then multiply it by the leftover budget
                    budget += int(leftover * s.data.size / (total_size - baseused_size))
                step = int(np.ceil(s.data.size / float(budget)))
                x = s.data[:, 0][::-step][::-1]
                y = s.data[:, 1]
                y_low_pass = np.convolve(y, np.ones(shape=(step,), dtype=y.dtype) / float(step), mode='same')
                y = y_low_pass[::-step][::-1]
                s.data = np.array([x, y], dtype=s.data.dtype).T

            # decide on number of points between mean and max
            s_max = np.max(np.abs(s.data), axis=0)
            s_max = np.maximum(s_max, s_max * 0 + 0.01)
            digits = np.maximum(np.array([1, 1]), np.array([6, 6]) - np.floor(np.abs(np.log10(s_max))))
            s.data[:, 0] = np.round(s.data[:, 0], int(digits[0]))
            s.data[:, 1] = np.round(s.data[:, 1], int(digits[1]))

    plotly_obj["data"].extend({
        "name": s.name,
        "x": s.data[:, 0].tolist(),
        "y": s.data[:, 1].tolist(),
        "mode": mode,
        "text": s.labels,
        "type": "scatter",
    } for s in series)

    if layout_config:
        plotly_obj["layout"] = merge_dicts(plotly_obj["layout"], layout_config)

    return plotly_obj


def create_2d_scatter_series(np_row_wise, title="Scatter", series_name="Series", xtitle="x", ytitle="y", mode="lines",
                             labels=None, comment=None, layout_config=None):
    """
    Create a 2D scatter Plotly graph from a 2 column numpy array
    :param np_row_wise: 2 column numpy data array [(x0,y0), (x1,y1) ...]
    :param title: Chart title
    :param series_name: Series name
    :param xtitle: X-axis title
    :param ytitle: Y-axis title
    :param mode: scatter type mode ('lines' / 'markers' / 'lines+markers')
    :param labels: label (text) per point on the scatter graph
    :param comment: comment underneath the title
    :param layout_config: optional dictionary for layout configuration, passed directly to plotly
    :return: Plotly chart dict.
    """
    plotly_obj = _plotly_scatter_layout_dict(  # noqa: F841
        title=title, xaxis_title=xtitle, yaxis_title=ytitle, comment=comment)
    assert np_row_wise.ndim == 2, "Expected a 2D numpy array"
    assert np_row_wise.shape[1] == 2, "Expected two columns X/Y e.g. [(x0,y0), (x1,y1) ...]"

    # this_scatter_data = {
    #     "name": series_name,
    #     "x": np_row_wise[:, 0].tolist(),
    #     "y": np_row_wise[:, 1].tolist(),
    #     "mode": mode,
    #     "text": labels,
    #     "type": "scatter"
    # }
    # plotly_obj["data"].append(this_scatter_data)
    # return plotly_obj
    series = SeriesInfo(name=series_name, data=np_row_wise, labels=labels)

    return create_line_plot(title=title, series=[series], xtitle=xtitle, ytitle=ytitle, mode=mode,
                            comment=comment, MAX_SIZE=100000, layout_config=layout_config)


def create_3d_scatter_series(np_row_wise, title="Scatter", series_name="Series", xtitle="x", ytitle="y", ztitle="z",
                             mode="lines", color=((217, 217, 217, 0.14),), marker_size=5, line_width=0.8,
                             labels=None, fill_axis=-1, plotly_obj=None, layout_config=None):
    """
    Create a 3D scatter Plotly graph from a 3 column numpy array
    :param np_row_wise: 3 column numpy data array [(x0,y0,z0), (x1,y1,z1) ...]
    :param title: Chart title
    :param series_name: Series name
    :param xtitle: X-axis title
    :param ytitle: Y-axis title
    :param ztitle: Z-axis title
    :param labels: label (text) per point on the scatter graph
    :param fill_axis: fill area under the curve
    :param layout_config: additional layout configuration
    :return: Plotly chart dict.
    """
    if not plotly_obj:
        plotly_obj = plotly_scatter3d_layout_dict(
            title=title, xaxis_title=xtitle, yaxis_title=ytitle, zaxis_title=ztitle)
    assert np_row_wise.ndim == 2, "Expected a 2D numpy array"
    assert np_row_wise.shape[1] == 3, "Expected three columns X/Y/Z e.g. [(x0,y0,z0), (x1,y1,z1) ...]"

    c = color[0]
    c = (int(c[0]), int(c[1]), int(c[2]), float(c[3]))
    this_scatter_data = {
        "name": series_name,
        "x": np_row_wise[:, 0].tolist(),
        "y": np_row_wise[:, 1].tolist(),
        "z": np_row_wise[:, 2].tolist(),
        "text": labels,
        "type": "scatter3d",
        "mode": mode,
        'marker': {
            'size': marker_size,
            'line': {
                'color': 'rgba(%d, %d, %d, %f.2)' % (c[0], c[1], c[2], c[3]),
                'width': line_width
            },
            'opacity': 0.8
        },
    }
    plotly_obj["data"].append(this_scatter_data)

    if layout_config:
        plotly_obj["layout"] = merge_dicts(plotly_obj["layout"], layout_config)

    return plotly_obj


def create_value_matrix(np_value_matrix, title="Heatmap Matrix", xlabels=None, ylabels=None, xtitle="X", ytitle="Y",
                        custom_colors=True, series=None, comment=None, yaxis_reversed=False, layout_config=None):
    conf_matrix_plot = {
        "data": [
            {
                "x": xlabels,
                "y": ylabels,
                "z": np_value_matrix.tolist(),
                "type": "heatmap"
            }
        ],
        "layout": {
            "showlegend": True,
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),

            "xaxis": {
                "title": xtitle,
            },
            "yaxis": {
                "title": ytitle
            },
            "name": series,
        }
    }
    if yaxis_reversed:
        conf_matrix_plot['layout']['yaxis']['autorange'] = "reversed"

    if custom_colors and not layout_config:
        scale, bar = _get_z_colorbar_data()
        conf_matrix_plot["data"][0].update({"colorscale": scale})
        conf_matrix_plot["data"][0].update({"colorbar": bar})

    if layout_config:
        conf_matrix_plot["data"][0] = merge_dicts(conf_matrix_plot["data"][0], layout_config)
        conf_matrix_plot["layout"] = merge_dicts(conf_matrix_plot["layout"], layout_config)

    return conf_matrix_plot


def create_3d_surface(np_value_matrix, title="3D Surface", xlabels=None, ylabels=None, xtitle="X", ytitle="Y",
                      ztitle="Z", custom_colors=True, series=None, camera=None, comment=None, layout_config=None):
    surface_plot = {
        "data": [
            {
                "z": np_value_matrix.tolist(),
                "type": "surface",
                "contours": {
                    "y": {
                        "show": False,
                        "highlightcolor": "#fff4ff",
                        "project": {"y": True}
                    }
                },
                "showscale": False,
            }
        ],
        "layout": {
            "scene": {
                "xaxis": {
                    "title": xtitle,
                    "showgrid": False,
                    "nticks": 10,
                    "ticktext": xlabels,
                    "tickvals": list(range(len(xlabels))) if xlabels else None,
                },
                "yaxis": {
                    "title": ytitle,
                    "showgrid": False,
                    "nticks": 10,
                    "ticktext": ylabels,
                    "tickvals": list(range(len(ylabels))) if ylabels else ylabels,
                },
                "zaxis": {
                    "title": ztitle,
                    "nticks": 5,
                },
            },
            "showlegend": False,
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "name": series,
        }
    }
    if camera:
        surface_plot['layout']['scene']['camera'] = {"eye": {"x": camera[0], "y": camera[1], "z": camera[2]}}

    if custom_colors:
        scale, bar = _get_z_colorbar_data()
        surface_plot["data"][0].update({"colorscale": scale})
        surface_plot["data"][0].update({"colorbar": bar})

    if layout_config:
        surface_plot["layout"] = merge_dicts(surface_plot["layout"], layout_config)

    return surface_plot


def create_image_plot(image_src, title, width=640, height=480, series=None, comment=None, layout_config=None):
    image_plot = {
        "data": [],
        "layout": {
            "xaxis": {"visible": False, "range": [0, width]},
            "yaxis": {"visible": False, "range": [0, height], "scaleanchor": "x"},
            # "width": width,
            # "height": height,
            "margin": {'l': 0, 'r': 0, 't': 64, 'b': 0},
            "images": [{
                "sizex": width,
                "sizey": height,
                "xref": "x",
                "yref": "y",
                "opacity": 1.0,
                "x": 0,
                "y": height,
                # "xanchor": "left",
                # "yanchor": "bottom",
                "sizing": "contain",
                "layer": "below",
                "source": image_src
            }],
            "showlegend": False,
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "name": series,
        }
    }

    if layout_config:
        image_plot["layout"] = merge_dicts(image_plot["layout"], layout_config)

    return image_plot


def _get_z_colorbar_data(z_data=None, values=None, colors=None):
    if values is None:
        values = [0, 1. / 10, 2. / 10, 6. / 10, 9. / 10]
    if colors is None:
        colors = [(71, 17, 100), (53, 92, 140), (37, 139, 141), (66, 189, 112), (141, 314, 68), (221, 226, 24)]
    if z_data is not None:
        data = np.array(z_data)
        max_z = data.max()
        scaler = max_z
        values = [float(v * scaler) for v in values[0:5]]
    values.append(1.0)  # poltly quirk?
    # we do not want to show the first and last value
    tickvalues = [" %.3f " % v for v in values[1:]]
    tickvalues = [float(v) for v in tickvalues]
    # tickvalues.pop()
    colorscale = [[v, 'rgb' + str(color)] for v, color in zip(values, colors)]
    colorbar = {"tick0": 0, "tickmode": "array", "tickvals": tickvalues}

    return colorscale, colorbar


def _plotly_hist_dict(title, xtitle, ytitle, mode='group', data=None, comment=None, layout_config=None):
    """
    Create a basic Plotly chart dictionary
    :param title: Chart title
    :param xtitle: X-Series title
    :param ytitle: Y-Series title
    :param mode: multiple histograms mode. optionals stack / group / relative. Default is 'group'.
    :param data: Data items
    :type data: list
    :param layout_config: dict
    :return: Plotly chart dict.
    """
    assert mode in ('stack', 'group', 'relative')

    plotly_object = {
        "data": data or [],
        "layout": {
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "xaxis": {
                "title": xtitle
            },
            "yaxis": {
                "title": ytitle
            },
            "barmode": mode,
            "bargap": 0.08,
            "bargroupgap": 0
        }
    }
    if layout_config:
        plotly_object["layout"] = merge_dicts(plotly_object["layout"], layout_config)

    return plotly_object


def _np_row_to_plotly_data_item(np_row, label, xlabels=None):
    """
    Convert a numpy data row into a Plotly chart data item
    :param np_row: numpy 1D data row
    :param label: Item label
    :return: A plotly data item dict.
    """
    bins = list(range(np_row.shape[0])) if xlabels is None else list(xlabels)
    # mylabels = ['"' + label + '"'] * len(bins)
    this_trace_data = {
        "name": label,
        "y": np_row.tolist(),
        "x": bins,
        # "text": mylabels,
        "type": "bar"
    }
    return this_trace_data


def _plotly_scatter_layout_dict(title="Scatter", xaxis_title="X", yaxis_title="Y", series=None, comment=None):
    return {
        "data": [],
        "layout": {
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "xaxis": {
                "title": xaxis_title,
                "showspikes": True,
                "spikethickness": 1,
                "spikesnap": "cursor",
                "spikemode": "toaxis+across",
            },
            "yaxis": {
                "title": yaxis_title,
                "showspikes": True,
                "spikethickness": 1,
                "spikesnap": "cursor",
                "spikemode": "toaxis+across",
            },
            "name": series,
        }
    }


def plotly_scatter3d_layout_dict(title="Scatter", xaxis_title="X", yaxis_title="Y", zaxis_title="Z",
                                 series=None, show_legend=True, comment=None, layout_config=None):
    plotly_object = {
        "data": [],
        "layout": {
            "showlegend": show_legend,
            "title": title if not comment else (title + '<br><sup>' + comment + '</sup>'),
            "scene": {
                'xaxis': {'title': xaxis_title},
                'yaxis': {'title': yaxis_title},
                'zaxis': {'title': zaxis_title},
            },
            "name": series,
        }
    }

    if layout_config:
        plotly_object["layout"] = merge_dicts(plotly_object["layout"], layout_config)

    return plotly_object


def create_plotly_table(table_plot, title, series, layout_config=None):
    """
    Create a basic Plotly table json style to be sent

    :param table_plot: the output table in pandas.DataFrame structure or list of rows (list) in a table
    :param title: Title (AKA metric)
    :type title: str
    :param series: Series (AKA variant)
    :type series: str
    :param layout_config: additional configuration layout
    :return: dict with plotly data.
    """
    is_list = isinstance(table_plot, (list, tuple))
    if is_list and (not table_plot or not any(table_plot)):
        # The list if empty
        headers_values = []
        cells_values = []
    elif is_list and table_plot[0] and isinstance(table_plot[0], (list, tuple)):
        headers_values = table_plot[0]
        cells_values = [list(i) for i in zip(*table_plot[1:])]
    else:
        if not pd:
            raise UsageError(
                "pandas is required in order to support reporting tables using CSV or a URL, "
                "please install the pandas python package"
            )
        index_added = not isinstance(table_plot.index, pd.RangeIndex)
        headers_values = list([col] for col in table_plot.columns)
        cells_values = json.loads(table_plot.T.to_json(orient='values', date_format='iso'))
        if index_added:
            if isinstance(table_plot.index, pd.MultiIndex):
                headers_values = [n or "" for n in (table_plot.index.names or [])] + headers_values
                cells_values = list(zip(*(table_plot.index.values.tolist() or []))) + cells_values
            else:
                headers_values.insert(0, table_plot.index.name or "")
                cells_values.insert(0, table_plot.index.values.tolist())

    ret = {
        "data": [{
            'type': 'table',
            'header': {
                'values': headers_values,
                'align': "left",
                'line': {'width': 0.5, 'color': '#d4d6e0'},
                'fill': {'color': "#fff"},
                'font': {'family': "Heebo, verdana, arial, sans-serif", 'size': 12, 'color': "#333"}
            },
            'cells': {
                'height': 30,
                'values': cells_values,
                'align': "left",
                'line': {'color': "white", 'width': 1},
                'font': {'family': "Heebo, verdana, arial, sans-serif", 'size': 14, 'color': "#384161"},
            }
        }],
        "layout": {
            "title.text": title,
            "title": series,
        }
    }
    if layout_config:
        ret["layout"] = merge_dicts(ret["layout"], layout_config)

    return ret


# clearml/utilities/proxy_object.py

import itertools
import json
from copy import copy

import six
import yaml


class ProxyDictPostWrite(dict):
    """ Dictionary wrapper that updates an arguments instance on any item set in the dictionary """

    def __init__(self, update_obj, update_func, *args, **kwargs):
        super(ProxyDictPostWrite, self).__init__(*args, **kwargs)
        self._update_obj = update_obj
        self._update_func = None
        for k, i in self.items():
            if isinstance(i, dict):
                super(ProxyDictPostWrite, self).update({k: ProxyDictPostWrite(update_obj, self._set_callback, i)})
        self._update_func = update_func

    def __setitem__(self, key, value):
        super(ProxyDictPostWrite, self).__setitem__(key, value)
        self._set_callback()

    def __reduce__(self):
        return dict, (), None, None, iter(self._to_dict().items())

    def _set_callback(self, *_):
        if self._update_func:
            self._update_func(self._update_obj, self)

    def _to_dict(self):
        a_dict = {}
        for k, i in self.items():
            if isinstance(i, ProxyDictPostWrite):
                a_dict[k] = i._to_dict()
            else:
                a_dict[k] = i
        return a_dict

    def update(self, E=None, **F):
        return super(ProxyDictPostWrite, self).update(
            ProxyDictPostWrite(self._update_obj, self._set_callback, E) if E is not None else
            ProxyDictPostWrite(self._update_obj, self._set_callback, **F))


class ProxyDictPreWrite(dict):
    """ Dictionary wrapper that prevents modifications to the dictionary """

    def __init__(self, update_obj, update_func, *args, **kwargs):
        super(ProxyDictPreWrite, self).__init__(*args, **kwargs)
        self._update_func = None
        for k, i in self.items():
            if isinstance(i, dict):
                self.update({k: ProxyDictPreWrite(k, self._nested_callback, i)})
        self._update_obj = update_obj
        self._update_func = update_func

    def __reduce__(self):
        return dict, (), None, None, iter(self.items())

    def __setitem__(self, key, value):
        key_value = self._set_callback((key, value,))
        if key_value:
            super(ProxyDictPreWrite, self).__setitem__(*key_value)

    def _set_callback(self, key_value, *_):
        if self._update_func is not None:
            if callable(self._update_func):
                res = self._update_func(self._update_obj, key_value)
            else:
                res = self._update_func

            if not res:
                return None
            return res
        return key_value

    def _nested_callback(self, prefix, key_value):
        return self._set_callback((prefix + '.' + key_value[0], key_value[1],))


def verify_basic_type(a_dict_list, basic_types=None):
    basic_types = (float, int, bool, six.string_types, ) if not basic_types else \
        tuple(b for b in basic_types if b not in (list, tuple, dict))

    if isinstance(a_dict_list, basic_types):
        return True
    if isinstance(a_dict_list, (list, tuple)):
        return all(verify_basic_type(v, basic_types=basic_types) for v in a_dict_list)
    elif isinstance(a_dict_list, dict):
        return all(verify_basic_type(k, basic_types=basic_types) for k in a_dict_list.keys()) and \
               all(verify_basic_type(v, basic_types=basic_types) for v in a_dict_list.values())


def convert_bool(s):
    s = s.strip().lower()
    if s == "true":
        return True
    elif s == "false" or not s:
        return False
    raise ValueError("Invalid value (boolean literal expected): {}".format(s))


def cast_basic_type(value, type_str):
    if not type_str:
        return value

    basic_types = {str(getattr(v, '__name__', v)): v for v in (float, int, str, list, tuple, dict)}
    basic_types['bool'] = convert_bool

    parts = type_str.split('/')
    # nested = len(parts) > 1

    if parts[0] in ('list', 'tuple'):
        v = '[' + value.lstrip('[(').rstrip('])') + ']'
        v = yaml.load(v, Loader=yaml.SafeLoader)
        return basic_types.get(parts[0])(v)
    elif parts[0] in ('dict', ):
        try:
            return json.loads(value)
        except Exception:
            pass
        return value

    t = basic_types.get(str(type_str).lower().strip(), False)
    if t is not False:
        # noinspection PyBroadException
        try:
            return t(value)
        except Exception:
            return value

    return value


def get_basic_type(value):
    basic_types = (float, int, bool, six.string_types, list, tuple, dict)

    if isinstance(value, (list, tuple)) and value:
        tv = type(value)
        t = type(value[0])
        if all(t == type(v) for v in value):
            return '{}/{}'.format(str(getattr(tv, '__name__', tv)), str(getattr(t, '__name__', t)))
    elif isinstance(value, dict) and value:
        t = type(list(value.values())[0])
        if all(t == type(v) for v in value.values()):
            return 'dict/{}'.format(str(getattr(t, '__name__', t)))

    # it might be an empty list/dict/tuple
    t = type(value)
    if isinstance(value, basic_types):
        return str(getattr(t, '__name__', t))

    # we are storing it, even though we will not be able to restore it
    return str(getattr(t, '__name__', t))


def flatten_dictionary(a_dict, prefix='', sep='/'):
    flat_dict = {}
    basic_types = (float, int, bool, six.string_types, )
    for k, v in a_dict.items():
        k = str(k)
        if isinstance(v, (float, int, bool, six.string_types)):
            flat_dict[prefix + k] = v
        elif isinstance(v, (list, tuple)) and all([isinstance(i, basic_types) for i in v]):
            flat_dict[prefix + k] = v
        elif isinstance(v, dict):
            nested_flat_dict = flatten_dictionary(v, prefix=prefix + k + sep, sep=sep)
            if nested_flat_dict:
                flat_dict.update(nested_flat_dict)
            else:
                flat_dict[k] = {}
        else:
            # this is a mixture of list and dict, or any other object,
            # leave it as is, we have nothing to do with it.
            flat_dict[prefix + k] = v
    return flat_dict


def nested_from_flat_dictionary(a_dict, flat_dict, prefix='', sep='/'):
    basic_types = (float, int, bool, six.string_types, )
    org_dict = copy(a_dict)
    for k, v in org_dict.items():
        k = str(k)
        if isinstance(v, (float, int, bool, six.string_types)):
            a_dict[k] = flat_dict.get(prefix + k, v)
        elif isinstance(v, (list, tuple)) and all([isinstance(i, basic_types) for i in v]):
            a_dict[k] = flat_dict.get(prefix + k, v)
        elif isinstance(v, dict):
            a_dict[k] = nested_from_flat_dictionary(v, flat_dict, prefix=prefix + k + sep, sep=sep) or v
        else:
            # this is a mixture of list and dict, or any other object,
            # leave it as is, we have nothing to do with it.
            a_dict[k] = flat_dict.get(prefix + k, v)
    return a_dict


def naive_nested_from_flat_dictionary(flat_dict, sep='/'):
    """ A naive conversion of a flat dictionary with '/'-separated keys signifying nesting
    into a nested dictionary.
    """
    return {
        sub_prefix: (
            bucket[0][1] if (len(bucket) == 1 and sub_prefix == bucket[0][0])
            else naive_nested_from_flat_dictionary(
                {
                    k[len(sub_prefix) + 1:]: v
                    for k, v in bucket
                    if len(k) > len(sub_prefix)
                }, sep=sep
            )
        )
        for sub_prefix, bucket in (
            (key, list(group))
            for key, group in itertools.groupby(
                sorted(flat_dict.items()),
                key=lambda item: item[0].partition(sep)[0]
            )
        )
    }


def walk_nested_dict_tuple_list(dict_list_tuple, callback):
    # Do Not Change, type call will not trigger the auto resolving / download of the Lazy evaluator
    nested = (dict, tuple, list)
    type_dict_list_tuple = type(dict_list_tuple)
    if type_dict_list_tuple not in nested:
        return callback(dict_list_tuple)

    if type_dict_list_tuple == dict:
        ret = {}
        for k, v in dict_list_tuple.items():
            ret[k] = walk_nested_dict_tuple_list(v, callback=callback) if type(v) in nested else callback(v)

    else:
        ret = []
        for v in dict_list_tuple:
            ret.append(walk_nested_dict_tuple_list(v, callback=callback) if type(v) in nested else callback(v))

        if type_dict_list_tuple == tuple:
            ret = tuple(dict_list_tuple)

    return ret


class WrapperBase(type):

    # This metaclass is heavily inspired by the Object Proxying python recipe
    # (http://code.activestate.com/recipes/496741/). It adds special methods
    # to the wrapper class so it can proxy the wrapped class. In addition, it
    # adds a field __overrides__ in the wrapper class dictionary, containing
    # all attributes decorated to be overriden.

    _special_names = [
        '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
        '__contains__', '__delitem__', '__delslice__', '__div__', '__divmod__',
        '__eq__', '__float__', '__floordiv__', '__ge__', '__getitem__',
        '__getslice__', '__gt__', '__hash__', '__hex__', '__iadd__', '__iand__',
        '__idiv__', '__idivmod__', '__ifloordiv__', '__ilshift__', '__imod__',
        '__imul__', '__int__', '__invert__', '__ior__', '__ipow__', '__irshift__',
        '__isub__', '__iter__', '__itruediv__', '__ixor__', '__le__', '__len__',
        '__long__', '__lshift__', '__lt__', '__mod__', '__mul__', '__ne__',
        '__neg__', '__oct__', '__or__', '__pos__', '__pow__', '__radd__',
        '__rand__', '__rdiv__', '__rdivmod__', '__reduce__', '__reduce_ex__',
        '__repr__', '__reversed__', '__rfloorfiv__', '__rlshift__', '__rmod__',
        '__rmul__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
        '__rtruediv__', '__rxor__', '__setitem__', '__setslice__', '__sub__',
        '__truediv__', '__xor__', 'next', '__str__', '__repr__',
    ]

    def __new__(mcs, classname, bases, attrs):
        def make_method(name):
            def method(self, *args, **kwargs):
                obj = object.__getattribute__(self, "_wrapped")
                if obj is None:
                    cb = object.__getattribute__(self, "_callback")
                    obj = cb()
                    object.__setattr__(self, '_wrapped', obj)

                # we have to convert the instance to the real type
                if args and len(args) == 1 and (
                        type(args[0]) == LazyEvalWrapper or hasattr(type(args[0]), '_base_class_')):
                    try:
                        int(args[0])  # force loading the instance
                    except:  # noqa
                        pass
                    args = (object.__getattribute__(args[0], "_wrapped"), )

                mtd = getattr(obj, name)
                return mtd(*args, **kwargs)
            return method

        typed_class = attrs.get('_base_class_')
        for name in mcs._special_names:
            if not typed_class or hasattr(typed_class, name):
                attrs[name] = make_method(name)

        overrides = attrs.get('__overrides__', [])
        # overrides.extend(k for k, v in attrs.items() if isinstance(v, lazy))
        attrs['__overrides__'] = overrides
        return type.__new__(mcs, classname, bases, attrs)


class LazyEvalWrapper(six.with_metaclass(WrapperBase)):

    # This class acts as a proxy for the wrapped instance it is passed. All
    # access to its attributes are delegated to the wrapped class, except
    # those contained in __overrides__.

    __slots__ = ['_wrapped', '_callback', '_remote_reference', '__weakref__']

    _remote_reference_calls = []

    def __init__(self, callback, remote_reference=None):
        object.__setattr__(self, '_wrapped', None)
        object.__setattr__(self, '_callback', callback)
        object.__setattr__(self, '_remote_reference', remote_reference)
        if remote_reference:
            LazyEvalWrapper._remote_reference_calls.append(remote_reference)

    def _remoteref(self):
        func = object.__getattribute__(self, "_remote_reference")
        if func and func in LazyEvalWrapper._remote_reference_calls:
            LazyEvalWrapper._remote_reference_calls.remove(func)

        return func() if callable(func) else func

    def __getattribute__(self, attr):
        if attr in ('__isabstractmethod__', ):
            return None
        if attr in ('_remoteref', '_remote_reference'):
            return object.__getattribute__(self, attr)
        return getattr(LazyEvalWrapper._load_object(self), attr)

    def __setattr__(self, attr, value):
        setattr(LazyEvalWrapper._load_object(self), attr, value)

    def __delattr__(self, attr):
        delattr(LazyEvalWrapper._load_object(self), attr)

    def __nonzero__(self):
        return bool(LazyEvalWrapper._load_object(self))

    def __bool__(self):
        return bool(LazyEvalWrapper._load_object(self))

    @staticmethod
    def _load_object(self):
        obj = object.__getattribute__(self, "_wrapped")
        if obj is None:
            cb = object.__getattribute__(self, "_callback")
            obj = cb()
            object.__setattr__(self, '_wrapped', obj)
        return obj

    @classmethod
    def trigger_all_remote_references(cls):
        for func in cls._remote_reference_calls:
            if callable(func):
                func()
        cls._remote_reference_calls = []


def lazy_eval_wrapper_spec_class(class_type):
    class TypedLazyEvalWrapper(six.with_metaclass(WrapperBase)):
        _base_class_ = class_type
        __slots__ = ['_wrapped', '_callback', '__weakref__']

        def __init__(self, callback):
            object.__setattr__(self, '_wrapped', None)
            object.__setattr__(self, '_callback', callback)

        def __nonzero__(self):
            return bool(LazyEvalWrapper._load_object(self))

        def __bool__(self):
            return bool(LazyEvalWrapper._load_object(self))

        def __getattribute__(self, attr):
            if attr == '__isabstractmethod__':
                return None
            if attr == '__class__':
                return class_type

            return getattr(LazyEvalWrapper._load_object(self), attr)

    return TypedLazyEvalWrapper


# clearml/utilities/py3_interop.py

""" Convenience classes supporting python3-like concepts """
import abc

import six


@six.add_metaclass(abc.ABCMeta)
class AbstractContextManager(object):
    """An abstract base class for context managers. Supported in contextlib from python 3.6 and up """

    def __enter__(self):
        """Return `self` upon entering the runtime context."""
        return self

    @abc.abstractmethod
    def __exit__(self, exc_type, exc_value, traceback):
        """Raise any exception triggered within the runtime context."""
        return None

    @classmethod
    def __subclasshook__(cls, C):
        if cls is AbstractContextManager:
            if (any("__enter__" in B.__dict__ for B in C.__mro__) and any("__exit__" in B.__dict__ for B in C.__mro__)):
                return True
        return NotImplemented


try:

    from abc import abstractclassmethod

except ImportError:

    class abstractclassmethod(classmethod):
        __isabstractmethod__ = True

        def __init__(self, callable):
            callable.__isabstractmethod__ = True
            super(abstractclassmethod, self).__init__(callable)


# clearml/utilities/resource_monitor.py

import logging
import os
import platform
import sys
import warnings
from time import time

import psutil
from pathlib2 import Path
from typing import Text

from .process.mp import BackgroundMonitor
from ..backend_api import Session
from ..binding.frameworks.tensorflow_bind import IsTensorboardInit

try:
    from .gpu import gpustat
except ImportError:
    gpustat = None


class ResourceMonitor(BackgroundMonitor):
    _title_machine = ':monitor:machine'
    _title_gpu = ':monitor:gpu'

    def __init__(self, task, sample_frequency_per_sec=2., report_frequency_sec=30.,
                 first_report_sec=None, wait_for_first_iteration_to_start_sec=180.0,
                 max_wait_for_first_iteration_to_start_sec=1800., report_mem_used_per_process=True):
        super(ResourceMonitor, self).__init__(task=task, wait_period=sample_frequency_per_sec)
        self._task = task
        self._sample_frequency = sample_frequency_per_sec
        self._report_frequency = report_frequency_sec
        self._first_report_sec = first_report_sec or report_frequency_sec
        self.wait_for_first_iteration = wait_for_first_iteration_to_start_sec
        self.max_check_first_iteration = max_wait_for_first_iteration_to_start_sec
        self._num_readouts = 0
        self._readouts = {}
        self._previous_readouts = {}
        self._previous_readouts_ts = time()
        self._gpustat_fail = 0
        self._gpustat = gpustat
        self._active_gpus = None
        self._process_info = psutil.Process() if report_mem_used_per_process else None
        self._last_process_pool = {}
        self._last_process_id_list = []
        if not self._gpustat:
            self._task.get_logger().report_text('ClearML Monitor: GPU monitoring is not available')
        else:  # if running_remotely():
            # noinspection PyBroadException
            try:
                active_gpus = os.environ.get('NVIDIA_VISIBLE_DEVICES', '') or \
                    os.environ.get('CUDA_VISIBLE_DEVICES', '')
                if active_gpus:
                    self._active_gpus = [int(g.strip()) for g in active_gpus.split(',')]
            except Exception:
                pass

    def daemon(self):
        if self._is_thread_mode_and_not_main_process():
            return

        seconds_since_started = 0
        reported = 0
        last_iteration = 0
        fallback_to_sec_as_iterations = None

        # get max GPU ID, and make sure our active list is within range
        if self._active_gpus:
            # noinspection PyBroadException
            try:
                gpu_stat = self._gpustat.new_query()
                if max(self._active_gpus) > len(gpu_stat.gpus) - 1:
                    self._active_gpus = None
            except Exception:
                pass

        # add Task runtime_properties with the machine spec
        if Session.check_min_api_version('2.13'):
            try:
                machine_spec = self._get_machine_specs()
                if machine_spec:
                    # noinspection PyProtectedMember
                    self._task._set_runtime_properties(runtime_properties=machine_spec)
            except Exception as ex:
                logging.getLogger('clearml.resource_monitor').debug(
                    'Failed logging machine specification: {}'.format(ex))

        # last_iteration_interval = None
        # last_iteration_ts = 0
        # repeated_iterations = 0
        while True:
            last_report = time()
            current_report_frequency = self._report_frequency if reported != 0 else self._first_report_sec
            while (time() - last_report) < current_report_frequency:
                # wait for self._sample_frequency seconds, if event set quit
                if self._event.wait(1.0 / self._sample_frequency):
                    return
                # noinspection PyBroadException
                try:
                    self._update_readouts()
                except Exception:
                    pass

            seconds_since_started += int(round(time() - last_report))
            # check if we do not report any metric (so it means the last iteration will not be changed)
            if fallback_to_sec_as_iterations is None:
                if IsTensorboardInit.tensorboard_used():
                    fallback_to_sec_as_iterations = False
                elif seconds_since_started >= self.wait_for_first_iteration:
                    self._task.get_logger().report_text('ClearML Monitor: Could not detect iteration reporting, '
                                                        'falling back to iterations as seconds-from-start')
                    fallback_to_sec_as_iterations = True
            elif fallback_to_sec_as_iterations is True and seconds_since_started <= self.max_check_first_iteration:
                if self._check_logger_reported():
                    fallback_to_sec_as_iterations = False
                    self._task.get_logger().report_text('ClearML Monitor: Reporting detected, '
                                                        'reverting back to iteration based reporting')

            clear_readouts = True
            # if we do not have last_iteration, we just use seconds as iteration
            if fallback_to_sec_as_iterations:
                iteration = seconds_since_started
            else:
                iteration = self._task.get_last_iteration()
                if iteration < last_iteration:
                    # we started a new session?!
                    # wait out
                    clear_readouts = False
                    iteration = last_iteration
                elif iteration == last_iteration:
                    # repeated_iterations += 1
                    # if last_iteration_interval:
                    #     # to be on the safe side, we don't want to pass the actual next iteration
                    #     iteration += int(0.95*last_iteration_interval[0] * (seconds_since_started - last_iteration_ts)
                    #                      / last_iteration_interval[1])
                    # else:
                    #     iteration += 1
                    clear_readouts = False
                    iteration = last_iteration
                else:
                    # last_iteration_interval = (iteration - last_iteration, seconds_since_started - last_iteration_ts)
                    # repeated_iterations = 0
                    # last_iteration_ts = seconds_since_started
                    last_iteration = iteration
                    fallback_to_sec_as_iterations = False
                    clear_readouts = True

            # start reporting only when we figured out, if this is seconds based, or iterations based
            average_readouts = self._get_average_readouts()
            if fallback_to_sec_as_iterations is not None:
                for k, v in average_readouts.items():
                    # noinspection PyBroadException
                    try:
                        title = self._title_gpu if k.startswith('gpu_') else self._title_machine
                        # 3 points after the dot
                        value = round(v * 1000) / 1000.
                        self._task.get_logger().report_scalar(title=title, series=k, iteration=iteration, value=value)
                    except Exception:
                        pass
                # clear readouts if this is update is not averaged
                if clear_readouts:
                    self._clear_readouts()

            # count reported iterations
            reported += 1

    def _update_readouts(self):
        readouts = self._machine_stats()
        elapsed = time() - self._previous_readouts_ts
        self._previous_readouts_ts = time()
        for k, v in readouts.items():
            # cumulative measurements
            if k.endswith('_mbs'):
                v = (v - self._previous_readouts.get(k, v)) / elapsed

            self._readouts[k] = self._readouts.get(k, 0.0) + v
        self._num_readouts += 1
        self._previous_readouts = readouts

    def _get_num_readouts(self):
        return self._num_readouts

    def _get_average_readouts(self):
        average_readouts = dict((k, v / float(self._num_readouts)) for k, v in self._readouts.items())
        return average_readouts

    def _clear_readouts(self):
        self._readouts = {}
        self._num_readouts = 0

    def _machine_stats(self):
        """
        :return: machine stats dictionary, all values expressed in megabytes
        """
        cpu_usage = [float(v) for v in psutil.cpu_percent(percpu=True)]
        stats = {
            "cpu_usage": sum(cpu_usage) / float(len(cpu_usage)),
        }

        bytes_per_megabyte = 1024 ** 2

        def bytes_to_megabytes(x):
            return x / bytes_per_megabyte

        virtual_memory = psutil.virtual_memory()
        # stats["memory_used_gb"] = bytes_to_megabytes(virtual_memory.used) / 1024
        stats["memory_used_gb"] = bytes_to_megabytes(
            self._get_process_used_memory() if self._process_info else virtual_memory.used) / 1024
        stats["memory_free_gb"] = bytes_to_megabytes(virtual_memory.available) / 1024
        disk_use_percentage = psutil.disk_usage(Text(Path.home())).percent
        stats["disk_free_percent"] = 100.0 - disk_use_percentage
        with warnings.catch_warnings():
            if logging.root.level > logging.DEBUG:  # If the logging level is bigger than debug, ignore
                # psutil.sensors_temperatures warnings
                warnings.simplefilter("ignore", category=RuntimeWarning)
            sensor_stat = (psutil.sensors_temperatures() if hasattr(psutil, "sensors_temperatures") else {})
        if "coretemp" in sensor_stat and len(sensor_stat["coretemp"]):
            stats["cpu_temperature"] = max([float(t.current) for t in sensor_stat["coretemp"]])

        # protect against permission issues
        # update cached measurements
        # noinspection PyBroadException
        try:
            net_stats = psutil.net_io_counters()
            stats["network_tx_mbs"] = bytes_to_megabytes(net_stats.bytes_sent)
            stats["network_rx_mbs"] = bytes_to_megabytes(net_stats.bytes_recv)
        except Exception:
            pass

        # protect against permission issues
        # noinspection PyBroadException
        try:
            io_stats = psutil.disk_io_counters()
            stats["io_read_mbs"] = bytes_to_megabytes(io_stats.read_bytes)
            stats["io_write_mbs"] = bytes_to_megabytes(io_stats.write_bytes)
        except Exception:
            pass

        # check if we can access the gpu statistics
        if self._gpustat:
            # noinspection PyBroadException
            try:
                stats.update(self._get_gpu_stats())
            except Exception:
                # something happened and we can't use gpu stats,
                self._gpustat_fail += 1
                if self._gpustat_fail >= 3:
                    self._task.get_logger().report_text('ClearML Monitor: GPU monitoring failed getting GPU reading, '
                                                        'switching off GPU monitoring')
                    self._gpustat = None

        return stats

    def _check_logger_reported(self):
        titles = self.get_logger_reported_titles(self._task)
        return len(titles) > 0

    @classmethod
    def get_logger_reported_titles(cls, task):
        # noinspection PyProtectedMember
        titles = list(task.get_logger()._get_used_title_series().keys())
        try:
            titles.remove(cls._title_machine)
        except ValueError:
            pass
        try:
            titles.remove(cls._title_gpu)
        except ValueError:
            pass
        return titles

    def _get_process_used_memory(self):
        def mem_usage_children(a_mem_size, pr, parent_mem=None):
            self._last_process_id_list.append(pr.pid)
            # add out memory usage
            our_mem = pr.memory_info()
            mem_diff = our_mem.rss - parent_mem.rss if parent_mem else our_mem.rss
            a_mem_size += mem_diff if mem_diff > 0 else 0
            # now we are the parent
            for child in pr.children():
                # get the current memory
                m = pr.memory_info()
                mem_diff = m.rss - our_mem.rss
                a_mem_size += mem_diff if mem_diff > 0 else 0
                a_mem_size = mem_usage_children(a_mem_size, child, parent_mem=m)
            return a_mem_size

        # only run the memory usage query once per reporting period
        # because this memory query is relatively slow, and changes very little.
        if self._last_process_pool.get('cpu') and \
                (time() - self._last_process_pool['cpu'][0]) < self._report_frequency:
            return self._last_process_pool['cpu'][1]

        # if we have no parent process, return 0 (it's an error)
        if not self._process_info:
            return 0

        self._last_process_id_list = []
        mem_size = mem_usage_children(0, self._process_info)
        self._last_process_pool['cpu'] = time(), mem_size

        return mem_size

    def _get_gpu_stats(self):
        if not self._gpustat:
            return {}

        # per process memory query id slow, so we only call it once per reporting period,
        # On the rest of the samples we return the previous memory measurement

        # update mem used by our process and sub processes
        if self._process_info and (not self._last_process_pool.get('gpu') or
                                   (time() - self._last_process_pool['gpu'][0]) >= self._report_frequency):
            gpu_stat = self._gpustat.new_query(per_process_stats=True)
            gpu_mem = {}
            for i, g in enumerate(gpu_stat.gpus):
                # only monitor the active gpu's, if none were selected, monitor everything
                if self._active_gpus and i not in self._active_gpus:
                    continue
                gpu_mem[i] = 0
                for p in g.processes:
                    if p['pid'] in self._last_process_id_list:
                        gpu_mem[i] += p.get('gpu_memory_usage', 0)
            self._last_process_pool['gpu'] = time(), gpu_mem
        else:
            # if we do no need to update the memory usage, run global query
            # if we have no parent process (backward compatibility), return global stats
            gpu_stat = self._gpustat.new_query()
            gpu_mem = self._last_process_pool['gpu'][1] if self._last_process_pool.get('gpu') else None

        # generate the statistics dict for actual report
        stats = {}
        for i, g in enumerate(gpu_stat.gpus):
            # only monitor the active gpu's, if none were selected, monitor everything
            if self._active_gpus and i not in self._active_gpus:
                continue
            stats["gpu_%d_temperature" % i] = float(g["temperature.gpu"])
            stats["gpu_%d_utilization" % i] = float(g["utilization.gpu"])
            stats["gpu_%d_mem_usage" % i] = 100. * float(g["memory.used"]) / float(g["memory.total"])
            # already in MBs
            stats["gpu_%d_mem_free_gb" % i] = float(g["memory.total"] - g["memory.used"]) / 1024
            # use previously sampled process gpu memory, or global if it does not exist
            stats["gpu_%d_mem_used_gb" % i] = float(gpu_mem[i] if gpu_mem else g["memory.used"]) / 1024

        return stats

    def _get_machine_specs(self):
        # type: () -> dict
        specs = {}
        # noinspection PyBroadException
        try:
            specs = {
                'platform': str(sys.platform),
                'python_version': str(platform.python_version()),
                'python_exec': str(sys.executable),
                'OS': str(platform.platform(aliased=True)),
                'processor': str(platform.machine()),
                'cpu_cores': int(psutil.cpu_count()),
                'memory_gb': round(psutil.virtual_memory().total / 1024 ** 3, 1),
                'hostname': str(platform.node()),
                'gpu_count': 0,
            }
            if self._gpustat:
                gpu_stat = self._gpustat.new_query(shutdown=True, get_driver_info=True)
                if gpu_stat.gpus:
                    gpus = [g for i, g in enumerate(gpu_stat.gpus) if not self._active_gpus or i in self._active_gpus]
                    specs.update(
                        gpu_count=int(len(gpus)),
                        gpu_type=', '.join(g.name for g in gpus),
                        gpu_memory=', '.join('{}GB'.format(round(g.memory_total/1024.0)) for g in gpus),
                        gpu_driver_version=gpu_stat.driver_version or '',
                        gpu_driver_cuda_version=gpu_stat.driver_cuda_version or '',
                    )

        except Exception:
            pass

        return specs


# clearml/utilities/seed.py

import sys
import random

try:
    import numpy as np
except Exception:
    np = None


def make_deterministic(seed=1337, cudnn_deterministic=False):
    """
    Ensure deterministic behavior across PyTorch using the provided random seed.
    This function makes sure that torch, numpy and random use the same random seed.

    When using clearml's task, call this function using the task's random seed like so:
        make_deterministic(task.get_random_seed())

    :param int seed: Seed number
    :param bool cudnn_deterministic: In order to make computations deterministic on your specific platform
    and PyTorch release, set this value to True. torch will only allow those CuDNN algorithms that are
    (believed to be) deterministic. This can have a performance impact (slower execution) depending on your model.
    """
    seed = int(seed) & 0xFFFFFFFF
    torch = sys.modules.get("torch")
    tf = sys.modules.get("tensorflow")

    if cudnn_deterministic:
        try:
            torch.backends.cudnn.deterministic = True
            torch.backends.cudnn.benchmark = False
        except Exception:
            pass

    random.seed(seed)

    if np is not None:
        np.random.seed(seed)

    if torch is not None:
        try:
            torch.manual_seed(seed)
            torch.cuda.manual_seed(seed)
        except Exception:
            pass

    if tf is not None:
        # reset graph state
        try:
            import tensorflow  # noqa: F401
            from tensorflow.python.eager.context import _context
            eager_mode_bypass = _context is None
        except Exception:
            eager_mode_bypass = False

        if not eager_mode_bypass:
            try:
                tf.compat.v1.set_random_seed(seed)
            except Exception:
                try:
                    tf.set_random_seed(seed)
                except Exception:
                    pass

            try:
                tf.compat.v1.random.set_random_seed(seed)
            except Exception:
                try:
                    tf.random.set_random_seed(seed)
                except Exception:
                    pass


# clearml/utilities/version.py

from __future__ import absolute_import, division, print_function

from copy import deepcopy
from attr import attrs, attrib

import re
import six

if six.PY3:
    from math import inf
else:
    inf = float('inf')


class InvalidVersion(ValueError):
    """
    An invalid version was found, users should refer to PEP 440.
    """


@attrs
class _Version:
    epoch = attrib()
    release = attrib()
    dev = attrib()
    pre = attrib()
    post = attrib()
    local = attrib()


class _BaseVersion(object):
    def __init__(self, key):
        self._key = key

    def __hash__(self):
        return hash(self._key)

    def __lt__(self, other):
        return self._compare(other, lambda s, o: s < o)

    def __le__(self, other):
        return self._compare(other, lambda s, o: s <= o)

    def __eq__(self, other):
        return self._compare(other, lambda s, o: s == o)

    def __ge__(self, other):
        return self._compare(other, lambda s, o: s >= o)

    def __gt__(self, other):
        return self._compare(other, lambda s, o: s > o)

    def __ne__(self, other):
        return self._compare(other, lambda s, o: s != o)

    def _compare(self, other, method):
        if not isinstance(other, _BaseVersion):
            return NotImplemented

        return method(self._key, other._key)


class Version(_BaseVersion):
    VERSION_PATTERN = r"""
        v?
        (?:
            (?:(?P<epoch>[0-9]+)!)?                           # epoch
            (?P<release>[0-9]+(?:\.[0-9]+)*)                  # release segment
            (?P<pre>                                          # pre-release
                [-_\.]?
                (?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview))
                [-_\.]?
                (?P<pre_n>[0-9]+)?
            )?
            (?P<post>                                         # post release
                (?:-(?P<post_n1>[0-9]+))
                |
                (?:
                    [-_\.]?
                    (?P<post_l>post|rev|r)
                    [-_\.]?
                    (?P<post_n2>[0-9]+)?
                )
            )?
            (?P<dev>                                          # dev release
                [-_\.]?
                (?P<dev_l>dev)
                [-_\.]?
                (?P<dev_n>[0-9]+)?
            )?
        )
        (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
    """
    _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE)
    _local_version_separators = re.compile(r"[\._-]")

    def __init__(self, version):
        # Validate the version and parse it into pieces
        match = self._regex.search(version)
        if not match:
            raise InvalidVersion("Invalid version: '{0}'".format(version))

        # Store the parsed out pieces of the version
        self._version = _Version(
            epoch=int(match.group("epoch")) if match.group("epoch") else 0,
            release=tuple(int(i) for i in match.group("release").split(".")),
            pre=self._parse_letter_version(match.group("pre_l"), match.group("pre_n")),
            post=self._parse_letter_version(
                match.group("post_l") or '', match.group("post_n1") or match.group("post_n2") or ''
            ),
            dev=self._parse_letter_version(match.group("dev_l") or '', match.group("dev_n") or ''),
            local=self._parse_local_version(match.group("local") or ''),
        )

        # Generate a key which will be used for sorting
        key = self._cmpkey(
            self._version.epoch,
            self._version.release,
            self._version.pre,
            self._version.post,
            self._version.dev,
            self._version.local,
        )

        super(Version, self).__init__(key)

    def __repr__(self):
        return "<Version({0})>".format(repr(str(self)))

    def __str__(self):
        parts = []

        # Epoch
        if self.epoch != 0:
            parts.append("{0}!".format(self.epoch))

        # Release segment
        parts.append(".".join(str(x) for x in self.release))

        # Pre-release
        if self.pre is not None:
            parts.append("".join(str(x) for x in self.pre))

        # Post-release
        if self.post is not None:
            parts.append(".post{0}".format(self.post))

        # Development release
        if self.dev is not None:
            parts.append(".dev{0}".format(self.dev))

        # Local version segment
        if self.local is not None:
            parts.append("+{0}".format(self.local))

        return "".join(parts)

    def get_next_version(self):
        def increment(part):
            if isinstance(part, int):
                return part + 1
            type_ = type(part)
            part = list(part)
            if isinstance(part[-1], int):
                part[-1] += 1
            return type_(part)

        next_version = deepcopy(self)
        if next_version._version.dev:
            next_version._version.dev = increment(next_version._version.dev)
        elif next_version._version.post:
            next_version._version.post = increment(next_version._version.post)
        elif next_version._version.pre:
            next_version._version.pre = increment(next_version._version.pre)
        elif next_version._version.release:
            next_version._version.release = increment(next_version._version.release)
        elif next_version._version.epoch:
            next_version._version.epoch = increment(next_version._version.epoch)
        return next_version

    @property
    def epoch(self):
        return self._version.epoch

    @property
    def release(self):
        return self._version.release

    @property
    def pre(self):
        return self._version.pre

    @property
    def post(self):
        return self._version.post[1] if self._version.post else None

    @property
    def dev(self):
        return self._version.dev[1] if self._version.dev else None

    @property
    def local(self):
        if self._version.local:
            return ".".join(str(x) for x in self._version.local)
        else:
            return None

    @property
    def public(self):
        return str(self).split("+", 1)[0]

    @property
    def base_version(self):
        parts = []

        # Epoch
        if self.epoch != 0:
            parts.append("{0}!".format(self.epoch))

        # Release segment
        parts.append(".".join(str(x) for x in self.release))

        return "".join(parts)

    @property
    def is_prerelease(self):
        return self.dev is not None or self.pre is not None

    @property
    def is_postrelease(self):
        return self.post is not None

    @property
    def is_devrelease(self):
        return self.dev is not None

    @staticmethod
    def _parse_letter_version(letter, number):
        if not letter and not number:
            return None
        if letter:
            # We consider there to be an implicit 0 in a pre-release if there is
            # not a numeral associated with it.
            if number is None:
                number = 0

            # We normalize any letters to their lower case form
            letter = letter.lower()

            # We consider some words to be alternate spellings of other words and
            # in those cases we want to normalize the spellings to our preferred
            # spelling.
            if letter == "alpha":
                letter = "a"
            elif letter == "beta":
                letter = "b"
            elif letter in ["c", "pre", "preview"]:
                letter = "rc"
            elif letter in ["rev", "r"]:
                letter = "post"

            return letter, int(number)
        if not letter and number:
            # We assume if we are given a number, but we are not given a letter
            # then this is using the implicit post release syntax (e.g. 1.0-1)
            letter = "post"

        return letter, int(number)

    @classmethod
    def _parse_local_version(cls, local):
        """
        Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve").
        """
        if local is not None:
            local = tuple(
                part.lower() if not part.isdigit() else int(part)
                for part in cls._local_version_separators.split(local)
            )
            if not local or not local[0]:
                return None
            return local
        return None

    @staticmethod
    def _cmpkey(epoch, release, pre, post, dev, local):
        # When we compare a release version, we want to compare it with all of the
        # trailing zeros removed. So we'll use a reverse the list, drop all the now
        # leading zeros until we come to something non zero, then take the rest
        # re-reverse it back into the correct order and make it a tuple and use
        # that for our sorting key.
        # release = tuple(
        #     reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release))))
        # )

        # Versions without a pre-release (except as noted above) should sort after
        # those with one.
        if not pre:
            pre = inf
        elif pre:
            pre = pre[1]

        # Versions without a post segment should sort before those with one.
        if not post:
            post = -inf
        else:
            post = post[1]

        # Versions without a development segment should sort after those with one.
        if not dev:
            dev = inf
        else:
            dev = dev[1]

        if not local:
            # Versions without a local segment should sort before those with one.
            local = inf
        else:
            # Versions with a local segment need that segment parsed to implement
            # the sorting rules in PEP440.
            # - Alpha numeric segments sort before numeric segments
            # - Alpha numeric segments sort lexicographically
            # - Numeric segments sort numerically
            # - Shorter versions sort before longer versions when the prefixes
            #   match exactly
            local = local[1]

        return epoch, release, pre, post, dev, local


# clearml/utilities/requests_toolbelt/_compat.py
"""Private module full of compatibility hacks.

Primarily this is for downstream redistributions of requests that unvendor
urllib3 without providing a shim.

.. warning::

    This module is private. If you use it, and something breaks, you were
    warned
"""
import sys

import requests

try:
    from requests.packages.urllib3 import fields
    from requests.packages.urllib3 import filepost
    from requests.packages.urllib3 import poolmanager
except ImportError:
    from urllib3 import fields
    from urllib3 import filepost
    from urllib3 import poolmanager

try:
    from requests.packages.urllib3.connection import HTTPConnection
    from requests.packages.urllib3 import connection
except ImportError:
    try:
        from urllib3.connection import HTTPConnection
        from urllib3 import connection
    except ImportError:
        HTTPConnection = None
        connection = None


if requests.__build__ < 0x020300:
    timeout = None
else:
    try:
        from requests.packages.urllib3.util import timeout
    except ImportError:
        from urllib3.util import timeout

if requests.__build__ < 0x021000:
    gaecontrib = None
else:
    try:
        from requests.packages.urllib3.contrib import appengine as gaecontrib
    except ImportError:
        from urllib3.contrib import appengine as gaecontrib

if requests.__build__ < 0x021200:
    PyOpenSSLContext = None
else:
    try:
        from requests.packages.urllib3.contrib.pyopenssl \
                import PyOpenSSLContext
    except ImportError:
        try:
            from urllib3.contrib.pyopenssl import PyOpenSSLContext
        except ImportError:
            PyOpenSSLContext = None

PY3 = sys.version_info > (3, 0)

if PY3:
    from collections.abc import Mapping, MutableMapping
    import queue
    from urllib.parse import urlencode, urljoin
else:
    from collections import Mapping, MutableMapping
    import Queue as queue
    from urllib import urlencode
    from urlparse import urljoin

try:
    basestring = basestring
except NameError:
    basestring = (str, bytes)


class HTTPHeaderDict(MutableMapping):
    """
    :param headers:
        An iterable of field-value pairs. Must not contain multiple field names
        when compared case-insensitively.

    :param kwargs:
        Additional field-value pairs to pass in to ``dict.update``.

    A ``dict`` like container for storing HTTP Headers.

    Field names are stored and compared case-insensitively in compliance with
    RFC 7230. Iteration provides the first case-sensitive key seen for each
    case-insensitive pair.

    Using ``__setitem__`` syntax overwrites fields that compare equal
    case-insensitively in order to maintain ``dict``'s api. For fields that
    compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
    in a loop.

    If multiple fields that are equal case-insensitively are passed to the
    constructor or ``.update``, the behavior is undefined and some will be
    lost.

    >>> headers = HTTPHeaderDict()
    >>> headers.add('Set-Cookie', 'foo=bar')
    >>> headers.add('set-cookie', 'baz=quxx')
    >>> headers['content-length'] = '7'
    >>> headers['SET-cookie']
    'foo=bar, baz=quxx'
    >>> headers['Content-Length']
    '7'
    """

    def __init__(self, headers=None, **kwargs):
        super(HTTPHeaderDict, self).__init__()
        self._container = {}
        if headers is not None:
            if isinstance(headers, HTTPHeaderDict):
                self._copy_from(headers)
            else:
                self.extend(headers)
        if kwargs:
            self.extend(kwargs)

    def __setitem__(self, key, val):
        self._container[key.lower()] = (key, val)
        return self._container[key.lower()]

    def __getitem__(self, key):
        val = self._container[key.lower()]
        return ', '.join(val[1:])

    def __delitem__(self, key):
        del self._container[key.lower()]

    def __contains__(self, key):
        return key.lower() in self._container

    def __eq__(self, other):
        if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
            return False
        if not isinstance(other, type(self)):
            other = type(self)(other)
        return ({k.lower(): v for k, v in self.itermerged()} ==
                {k.lower(): v for k, v in other.itermerged()})

    def __ne__(self, other):
        return not self.__eq__(other)

    if not PY3:  # Python 2
        iterkeys = MutableMapping.iterkeys
        itervalues = MutableMapping.itervalues

    __marker = object()

    def __len__(self):
        return len(self._container)

    def __iter__(self):
        # Only provide the originally cased names
        for vals in self._container.values():
            yield vals[0]

    def pop(self, key, default=__marker):
        """D.pop(k[,d]) -> v, remove specified key and return its value.

        If key is not found, d is returned if given, otherwise KeyError is
        raised.
        """
        # Using the MutableMapping function directly fails due to the private
        # marker.
        # Using ordinary dict.pop would expose the internal structures.
        # So let's reinvent the wheel.
        try:
            value = self[key]
        except KeyError:
            if default is self.__marker:
                raise
            return default
        else:
            del self[key]
            return value

    def discard(self, key):
        try:
            del self[key]
        except KeyError:
            pass

    def add(self, key, val):
        """Adds a (name, value) pair, doesn't overwrite the value if it already
        exists.

        >>> headers = HTTPHeaderDict(foo='bar')
        >>> headers.add('Foo', 'baz')
        >>> headers['foo']
        'bar, baz'
        """
        key_lower = key.lower()
        new_vals = key, val
        # Keep the common case aka no item present as fast as possible
        vals = self._container.setdefault(key_lower, new_vals)
        if new_vals is not vals:
            # new_vals was not inserted, as there was a previous one
            if isinstance(vals, list):
                # If already several items got inserted, we have a list
                vals.append(val)
            else:
                # vals should be a tuple then, i.e. only one item so far
                # Need to convert the tuple to list for further extension
                self._container[key_lower] = [vals[0], vals[1], val]

    def extend(self, *args, **kwargs):
        """Generic import function for any type of header-like object.
        Adapted version of MutableMapping.update in order to insert items
        with self.add instead of self.__setitem__
        """
        if len(args) > 1:
            raise TypeError("extend() takes at most 1 positional "
                            "arguments ({} given)".format(len(args)))
        other = args[0] if len(args) >= 1 else ()

        if isinstance(other, HTTPHeaderDict):
            for key, val in other.iteritems():
                self.add(key, val)
        elif isinstance(other, Mapping):
            for key in other:
                self.add(key, other[key])
        elif hasattr(other, "keys"):
            for key in other.keys():
                self.add(key, other[key])
        else:
            for key, value in other:
                self.add(key, value)

        for key, value in kwargs.items():
            self.add(key, value)

    def getlist(self, key):
        """Returns a list of all the values for the named field. Returns an
        empty list if the key doesn't exist."""
        try:
            vals = self._container[key.lower()]
        except KeyError:
            return []
        else:
            if isinstance(vals, tuple):
                return [vals[1]]
            else:
                return vals[1:]

    # Backwards compatibility for httplib
    getheaders = getlist
    getallmatchingheaders = getlist
    iget = getlist

    def __repr__(self):
        return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))

    def _copy_from(self, other):
        for key in other:
            val = other.getlist(key)
            if isinstance(val, list):
                # Don't need to convert tuples
                val = list(val)
            self._container[key.lower()] = [key] + val

    def copy(self):
        clone = type(self)()
        clone._copy_from(self)
        return clone

    def iteritems(self):
        """Iterate over all header lines, including duplicate ones."""
        for key in self:
            vals = self._container[key.lower()]
            for val in vals[1:]:
                yield vals[0], val

    def itermerged(self):
        """Iterate over all headers, merging duplicate ones together."""
        for key in self:
            val = self._container[key.lower()]
            yield val[0], ', '.join(val[1:])

    def items(self):
        return list(self.iteritems())

    @classmethod
    def from_httplib(cls, message):  # Python 2
        """Read headers from a Python 2 httplib message object."""
        # python2.7 does not expose a proper API for exporting multiheaders
        # efficiently. This function re-reads raw lines from the message
        # object and extracts the multiheaders properly.
        headers = []

        for line in message.headers:
            if line.startswith((' ', '\t')):
                key, value = headers[-1]
                headers[-1] = (key, value + '\r\n' + line.rstrip())
                continue

            key, value = line.split(':', 1)
            headers.append((key, value.strip()))

        return cls(headers)


__all__ = (
    'basestring',
    'connection',
    'fields',
    'filepost',
    'poolmanager',
    'timeout',
    'HTTPHeaderDict',
    'queue',
    'urlencode',
    'gaecontrib',
    'urljoin',
    'PyOpenSSLContext',
)
