  self.set_header('ETag', str_to_etag(str(local_path)))
        self.set_header('Last-Modified', mtime_to_str(local_path.stat().st_mtime))
        if local_path.is_file():
            self.set_header('Content-Length', local_path.stat().st_size)
        else:
            self.set_header('Content-Length', 0)
        await self.finish()

    async def get(self, ds_id: str, path: str = ''):
        key, local_path = self._get_key_and_local_path(ds_id, path)
        if _LOG_S3BUCKET_HANDLER:
            LOG.info(f'GET: key={key!r}, local_path={local_path!r}')
        if local_path is None or not local_path.exists():
            await self._key_not_found(key)
            return
        self.set_header('ETag', str_to_etag(str(local_path)))
        self.set_header('Last-Modified', mtime_to_str(local_path.stat().st_mtime))
        self.set_header('Content-Type', 'binary/octet-stream')
        if local_path.is_file():
            self.set_header('Content-Length', local_path.stat().st_size)
            chunk_size = 1024 * 1024
            with open(str(local_path), 'rb') as fp:
                while True:
                    chunk = fp.read(chunk_size)
                    if len(chunk) == 0:
                        break
                    self.write(chunk)
                    await self.flush()
        else:
            self.set_header('Content-Length', 0)
            await self.finish()

    def _key_not_found(self, key: str):
        self.set_header('Content-Type', 'application/xml')
        self.set_status(404)
        return self.finish(dict_to_xml('Error',
                                       dict(Code='NoSuchKey',
                                            Message='The specified key does not exist.',
                                            Key=key)))

    def _get_key_and_local_path(self, ds_id: str, path: str):
        dataset_config = self.service_context.get_dataset_config(ds_id)
        file_system = dataset_config.get('FileSystem', 'file')
        required_file_systems = ['file', 'local']
        if file_system not in required_file_systems:
            required_file_system_string = " or ".join(required_file_systems)
            raise ServiceBadRequestError(
                f'AWS S3 data access: currently, only datasets in file systems '
                f'{required_file_system_string!r} are supported, but dataset '
                f'{ds_id!r} uses file system {file_system!r}')

        key = f'{ds_id}/{path}'

        # validate path
        if path and '..' in path.split('/'):
            raise ServiceBadRequestError(f'AWS S3 data access: received illegal key {key!r}')

        bucket_mapping = self.service_context.get_s3_bucket_mapping()
        local_path = bucket_mapping.get(ds_id)
        local_path = os.path.join(local_path, path)

        local_path = os.path.normpath(local_path)

        return key, pathlib.Path(local_path)


# noinspection PyAbstractClass,PyBroadException
class GetWMTSTileHandler(ServiceRequestHandler):

    async def get(self, ds_id: str, var_name: str, z: str, y: str, x: str):
        self.set_caseless_query_arguments()
        tile = await IOLoop.current().run_in_executor(None,
                                                      get_dataset_tile,
                                                      self.service_context,
                                                      ds_id, var_name,
                                                      x, y, z,
                                                      self.params)
        self.set_header('Content-Type', 'image/png')
        await self.finish(tile)


# noinspection PyAbstractClass,PyBroadException
class GetDatasetVarTileHandler(ServiceRequestHandler):

    async def get(self, ds_id: str, var_name: str, z: str, x: str, y: str):
        tile = await IOLoop.current().run_in_executor(None,
                                                      get_dataset_tile,
                                                      self.service_context,
                                                      ds_id, var_name,
                                                      x, y, z,
                                                      self.params)
        self.set_header('Content-Type', 'image/png')
        await self.finish(tile)


# noinspection PyAbstractClass,PyBroadException
class GetDatasetVarTile2Handler(ServiceRequestHandler):

    async def get(self, ds_id: str, var_name: str, z: str, y: str, x: str):
        tile = await IOLoop.current().run_in_executor(None,
                                                      get_dataset_tile2,
                                                      self.service_context,
                                                      ds_id, var_name,
                                                      x, y, z,
                                                      self.params)
        self.set_header('Content-Type', 'image/png')
        await self.finish(tile)


# noinspection PyAbstractClass,PyBroadException
class GetDatasetVarLegendHandler(ServiceRequestHandler):

    async def get(self, ds_id: str, var_name: str):
        tile = await IOLoop.current().run_in_executor(None,
                                                      get_legend,
                                                      self.service_context,
                                                      ds_id, var_name,
                                                      self.params)
        self.set_header('Content-Type', 'image/png')
        await self.finish(tile)


# noinspection PyAbstractClass
class GetDatasetVarTileGridHandler(ServiceRequestHandler):

    def get(self, ds_id: str, var_name: str):
        tile_client = self.params.get_query_argument('tiles', "ol4")
        response = get_dataset_tile_grid(self.service_context,
                                         ds_id, var_name,
                                         tile_client, self.base_url)
        self.set_header('Content-Type', 'application/json')
        self.write(json.dumps(response, indent=2))


# noinspection PyAbstractClass
class GetColorBarsJsonHandler(ServiceRequestHandler):

    # noinspection PyShadowingBuiltins
    def get(self):
        mime_type = 'application/json'
        response = get_color_bars(self.service_context, mime_type)
        self.set_header('Content-Type', mime_type)
        self.write(response)


# noinspection PyAbstractClass
class GetColorBarsHtmlHandler(ServiceRequestHandler):

    # noinspection PyShadowingBuiltins
    def get(self):
        mime_type = 'text/html'
        response = get_color_bars(self.service_context, mime_type)
        self.set_header('Content-Type', mime_type)
        self.write(response)


# noinspection PyAbstractClass
class GetPlaceGroupsHandler(ServiceRequestHandler):

    # noinspection PyShadowingBuiltins
    def get(self):
        response = self.service_context.get_global_place_groups(self.base_url)
        self.set_header('Content-Type', "application/json")
        self.write(json.dumps(response, indent=2))


# noinspection PyAbstractClass
class FindPlacesHandler(ServiceRequestHandler):

    # noinspection PyShadowingBuiltins
    def get(self, place_group_id: str):
        query_expr = self.params.get_query_argument("query", None)
        geom_wkt = self.params.get_query_argument("geom", None)
        box_coords = self.params.get_query_argument("bbox", None)
        comb_op = self.params.get_query_argument("comb", "and")
        if geom_wkt and box_coords:
            raise ServiceBadRequestError('Only one of "geom" and "bbox" may be given')
        response = find_places(self.service_context,
                               place_group_id,
                               self.base_url,
                               geom_wkt=geom_wkt, box_coords=box_coords,
                               query_expr=query_expr, comb_op=comb_op)
        self.set_header('Content-Type', "application/json")
        self.write(json.dumps(response, indent=2))

    # noinspection PyShadowingBuiltins
    def post(self, place_group_id: str):
        query_expr = self.params.get_query_argument("query", None)
        comb_op = self.params.get_query_argument("comb", "and")
        geojson_obj = self.get_body_as_json_object()
        response = find_places(self.service_context,
                               place_group_id,
                               self.base_url,
                               geojson_obj=geojson_obj,
                               query_expr=query_expr, comb_op=comb_op)
        self.set_header('Content-Type', "application/json")
        self.write(json.dumps(response, indent=2))


# noinspection PyAbstractClass
class FindDatasetPlacesHandler(ServiceRequestHandler):

    # noinspection PyShadowingBuiltins
    def get(self, place_group_id: str, ds_id: str):
        query_expr = self.params.get_query_argument("query", None)
        comb_op = self.params.get_query_argument("comb", "and")
        response = find_dataset_places(self.service_context,
                                       place_group_id,
                                       ds_id,
                                       self.base_url,
                                       query_expr=query_expr,
                                       comb_op=comb_op)
        self.set_header('Content-Type', "application/json")
        self.write(json.dumps(response, indent=2))


# noinspection PyAbstractClass
class InfoHandler(ServiceRequestHandler):

    def get(self):
        config_time = timestamp_to_iso_string(datetime.datetime.fromtimestamp(self.service_context.config_mtime),
                                              freq="ms")
        server_time = timestamp_to_iso_string(datetime.datetime.now(), freq="ms")
        self.set_header('Content-Type', 'application/json')
        self.write(json.dumps(dict(name=SERVER_NAME,
                                   description=SERVER_DESCRIPTION,
                                   version=version,
                                   versions=get_xcube_versions(),
                                   configTime=config_time,
                                   serverTime=server_time,
                                   serverPID=os.getpid()),
                              indent=2))


# noinspection PyAbstractClass
class GetTimeSeriesHandler(ServiceRequestHandler):

    async def post(self, ds_id: str, var_name: str):
        geo_json_object = self.get_body_as_json_object("GeoJSON object")
        agg_methods = self.params.get_query_argument('aggMethods', default=None)
        agg_methods = agg_methods.split(',') if agg_methods else None
        start_date = self.params.get_query_argument_datetime('startDate', default=None)
        end_date = self.params.get_query_argument_datetime('endDate', default=None)
        max_valids = self.params.get_query_argument_int('maxValids', default=None)
        _check_max_valids(max_valids)

        result = await IOLoop.current().run_in_executor(None,
                                                        get_time_series,
                                                        self.service_context,
                                                        ds_id,
                                                        var_name,
                                                        geo_json_object,
                                                        agg_methods,
                                                        start_date,
                                                        end_date,
                                                        max_valids)
        self.set_header('Content-Type', 'application/json')
        await self.finish(dict(result=result))


# noinspection PyAbstractClass
class GetTsLegacyInfoHandler(ServiceRequestHandler):

    async def get(self):
        response = await IOLoop.current().run_in_executor(None, get_time_series_info, self.service_context)
        self.set_header('Content-Type', 'application/json')
        await self.finish(response)


# noinspection PyAbstractClass
class GetTsLegacyForPointHandler(ServiceRequestHandler):

    async def get(self, ds_id: str, var_name: str):
        lon = self.params.get_query_argument_float('lon')
        lat = self.params.get_query_argument_float('lat')
        start_date = self.params.get_query_argument_datetime('startDate', default=None)
        end_date = self.params.get_query_argument_datetime('endDate', default=None)
        max_valids = self.params.get_query_argument_int('maxValids', default=None)
        _check_max_valids(max_valids)

        response = await IOLoop.current().run_in_executor(None,
                                                          get_time_series_for_point,
                                                          self.service_context,
                                                          ds_id, var_name,
                                                          lon, lat,
                                                          start_date,
                                                          end_date,
                                                          max_valids)
        self.set_header('Content-Type', 'application/json')
        await self.finish(response)


# noinspection PyAbstractClass
class GetTsLegacyForGeometryHandler(ServiceRequestHandler):

    async def post(self, ds_id: str, var_name: str):
        start_date = self.params.get_query_argument_datetime('startDate', default=None)
        end_date = self.params.get_query_argument_datetime('endDate', default=None)
        incl_count = self.params.get_query_argument_int('inclCount', default=1) != 0
        incl_stdev = self.params.get_query_argument_int('inclStDev', default=0) != 0
        max_valids = self.params.get_query_argument_int('maxValids', default=None)
        _check_max_valids(max_valids)
        geometry = self.get_body_as_json_object("GeoJSON geometry")

        response = await IOLoop.current().run_in_executor(None,
                                                          get_time_series_for_geometry,
                                                          self.service_context,
                                                          ds_id, var_name,
                                                          geometry,
                                                          start_date, end_date,
                                                          incl_count, incl_stdev,
                                                          max_valids)
        self.set_header('Content-Type', 'application/json')
        await self.finish(response)


# noinspection PyAbstractClass
class GetTsLegacyForGeometriesHandler(ServiceRequestHandler):

    async def post(self, ds_id: str, var_name: str):
        start_date = self.params.get_query_argument_datetime('startDate', default=None)
        end_date = self.params.get_query_argument_datetime('endDate', default=None)
        incl_count = self.params.get_query_argument_int('inclCount', default=1) != 0
        incl_stdev = self.params.get_query_argument_int('inclStDev', default=0) != 0
        max_valids = self.params.get_query_argument_int('maxValids', default=None)
        _check_max_valids(max_valids)
        geometry_collection = self.get_body_as_json_object("GeoJSON geometry collection")

        response = await IOLoop.current().run_in_executor(None,
                                                          get_time_series_for_geometry_collection,
                                                          self.service_context,
                                                          ds_id, var_name,
                                                          geometry_collection,
                                                          start_date, end_date,
                                                          incl_count, incl_stdev,
                                                          max_valids)
        self.set_header('Content-Type', 'application/json')
        await self.finish(response)


# noinspection PyAbstractClass
class GetTsLegacyForFeaturesHandler(ServiceRequestHandler):

    async def post(self, ds_id: str, var_name: str):
        start_date = self.params.get_query_argument_datetime('startDate', default=None)
        end_date = self.params.get_query_argument_datetime('endDate', default=None)
        incl_count = self.params.get_query_argument_int('inclCount', default=1) != 0
        incl_stdev = self.params.get_query_argument_int('inclStDev', default=0) != 0
        max_valids = self.params.get_query_argument_int('maxValids', default=None)
        _check_max_valids(max_valids)
        feature_collection = self.get_body_as_json_object("GeoJSON feature collection")

        response = await IOLoop.current().run_in_executor(None,
                                                          get_time_series_for_feature_collection,
                                                          self.service_context,
                                                          ds_id, var_name,
                                                          feature_collection,
                                                          start_date, end_date,
                                                          incl_count, incl_stdev,
                                                          max_valids)
        self.set_header('Content-Type', 'application/json')
        await self.finish(response)


def _check_max_valids(max_valids):
    if not (max_valids is None or max_valids == -1 or max_valids > 0):
        raise ServiceBadRequestError('If given, query parameter "maxValids" must be -1 or positive')


# xcube/webapi/reqparams.py

# The MIT License (MIT)
# Copyright (c) 2019 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from abc import abstractmethod, ABCMeta
from typing import Optional, Tuple, Mapping

import numpy as np

from xcube.util.undefined import UNDEFINED
from xcube.webapi.errors import ServiceBadRequestError


class RequestParams(metaclass=ABCMeta):

    @classmethod
    def to_int(cls, name: str, value: str) -> int:
        """
        Convert str value to int.
        :param name: Name of the value
        :param value: The string value
        :return: The int value
        :raise: ServiceBadRequestError
        """
        if value is None:
            raise ServiceBadRequestError(f'Parameter "{name}" must be an integer, but none was given')
        try:
            return int(value)
        except ValueError as e:
            raise ServiceBadRequestError(f'Parameter "{name}" must be an integer, but was {value!r}') from e

    @classmethod
    def to_float(cls, name: str, value: str) -> float:
        """
        Convert str value to float.
        :param name: Name of the value
        :param value: The string value
        :return: The float value
        :raise: ServiceBadRequestError
        """
        if value is None:
            raise ServiceBadRequestError(f'Parameter "{name}" must be a number, but none was given')
        try:
            return float(value)
        except ValueError as e:
            raise ServiceBadRequestError(f'Parameter "{name}" must be a number, but was {value!r}') from e

    @classmethod
    def to_datetime(cls, name: str, value: str) -> np.datetime64:
        """
        Convert str value to date/time value.
        :param name: Name of the value
        :param value: The string value
        :return: The date/time value
        :raise: ServiceBadRequestError
        """
        if value is None:
            raise ServiceBadRequestError(f'Parameter "{name}" must be a date/time, but none was given')
        try:
            return np.datetime64(value)
        except ValueError as e:
            raise ServiceBadRequestError(f'Parameter "{name}" must be a date/time, but was {value!r}') from e

    @abstractmethod
    def get_query_arguments(self) -> Mapping[str, str]:
        """ Get representation as mapping. """
        pass

    @abstractmethod
    def get_query_argument(self,
                           name: str,
                           default: Optional[str] = UNDEFINED) -> Optional[str]:
        """
        Get query argument.
        :param name: Query argument name
        :param default: Default value.
        :return: the value or none
        :raise: ServiceBadRequestError
        """

    def get_query_argument_int(self,
                               name: str,
                               default: Optional[int] = UNDEFINED) -> Optional[int]:
        """
        Get query argument of type int.
        :param name: Query argument name
        :param default: Default value.
        :return: int value
        :raise: ServiceBadRequestError
        """
        value = self.get_query_argument(name, default=default)
        return self.to_int(name, value) if value is not None else default

    def get_query_argument_float(self,
                                 name: str,
                                 default: Optional[float] = UNDEFINED) -> Optional[float]:
        """
        Get query argument of type float.
        :param name: Query argument name
        :param default: Default value.
        :return: float value
        :raise: ServiceBadRequestError
        """
        value = self.get_query_argument(name, default=default)
        return self.to_float(name, value) if value is not None else default

    def get_query_argument_datetime(self,
                                    name: str,
                                    default: Optional[np.datetime64] = UNDEFINED) -> Optional[np.datetime64]:
        """
        Get query argument of type float.
        :param name: Query argument name
        :param default: Default value.
        :return: float value
        :raise: ServiceBadRequestError
        """
        value = self.get_query_argument(name, default=default)

        return self.to_datetime(name, value) if value is not None else default

    def get_query_argument_point(self,
                                 name: str,
                                 default: Optional[Tuple[float, float]] = UNDEFINED) -> Optional[Tuple[float, float]]:

        value = self.get_query_argument(name, default=default)
        if value is None:
            return default
        if not isinstance(value, str):
            return value
        try:
            x, y = map(float, value.split(','))
            return x, y
        except ValueError:
            raise ServiceBadRequestError(f"Parameter {name!r} parameter must be a point using format '<lon>,<lat>',"
                                         f" but was {value!r}")


# xcube/webapi/s3util.py

import datetime
import hashlib
import os.path
import os.path
from typing import Dict, Any, List, Tuple, Iterator


def list_s3_bucket_v2(bucket_mapping: Dict[str, str],
                      name: str = None,
                      delimiter: str = None,
                      prefix: str = None,
                      max_keys: int = None,
                      start_after: str = None,
                      continuation_token: str = None,
                      storage_class: str = None,
                      last_modified: str = None,
                      key_to_etag: bool = False) -> Dict:
    """
    Implements AWS GET Bucket (List Objects) Version 2
    (https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html)
    for the local filesystem.

    :param bucket_mapping: mapping from the bucket's top-level names to directory paths in the local file system
    :param name: The bucket name, defaults to "s3bucket"
    :param delimiter: A delimiter is a character you use to group keys.
           If you specify a prefix, all of the keys that contain the same string
           between the prefix and the first occurrence of the delimiter after
           the prefix are grouped under a single result element called CommonPrefixes.
           If you don't specify the prefix parameter, the substring starts at the
           beginning of the key.
           The keys that are grouped under the CommonPrefixes result element are not
           returned elsewhere in the response.
           Refer to AWS docs for details. No default.
    :param prefix: Limits the response to keys that begin with the specified prefix.
           You can use prefixes to separate a bucket into different groupings of keys.
           (You can think of using prefix to make groups in the same way you'd use
           a folder in a file system.)
           Refer to AWS docs for details. No default.
    :param max_keys: Sets the maximum number of keys returned in the response body.
           If you want to retrieve fewer than the default 1000 keys,
           you can add this to your request.
           The response might contain fewer keys, but it never contains more.
           If there are additional keys that satisfy the search criteria, but these
           keys were not returned because max-keys was exceeded, the response
           contains <IsTruncated>true</IsTruncated>.
           To return the additional keys, see NextContinuationToken.
           Refer to AWS docs for details. Defaults to 1000.
    :param start_after: If you want the API to return key names after a specific object key in your key space,
           you can add this parameter. Amazon S3 lists objects in UTF-8 character
           encoding in lexicographical order.
           Refer to AWS docs for details. No default.
    :param continuation_token: When the response to this API call is truncated (that is, the IsTruncated
           response element value is true), the response also includes the
           NextContinuationToken element. To list the next set of objects,
           you can use the NextContinuationToken element in the next request
           as the continuation-token.
           Refer to AWS docs for details. No default.
    :param storage_class: Refer to AWS docs for details. Defaults to "STANDARD".
    :param last_modified: For testing only: always use this value for the "LastModified" entry of results
    :param key_to_etag: For testing only: use key to produce MD5 hashes for the "ETag" entry of results
    :return: A dictionary that represents the contents of a "ListBucketResult". Refer to AWS docs for details.
    """
    name = name or 's3bucket'
    max_keys = max_keys or 1000
    start_after = None if continuation_token else start_after
    storage_class = storage_class or 'STANDARD'

    contents_list = []
    next_continuation_token = None
    continuation_token_seen = continuation_token is None
    start_key_seen = start_after is None
    common_prefixes_list = []
    common_prefixes_set = set()

    token = 0

    for key, path in list_s3_bucket_keys(bucket_mapping):

        token += 1

        if len(contents_list) == max_keys:
            next_continuation_token = token
            break

        if key == start_after:
            start_key_seen = True
            continue

        if token == continuation_token:
            continuation_token_seen = True

        if not continuation_token_seen or not start_key_seen:
            continue

        if prefix and not key.startswith(prefix):
            continue

        if delimiter:
            index = key.find(delimiter, len(prefix) if prefix else 0)
            if index >= 0:
                key = key[:index + len(delimiter)]
                if key not in common_prefixes_set:
                    common_prefixes_set.add(key)
                    common_prefixes_list.append(key)
                continue

        stat = os.stat(path)
        item = dict(Key=key,
                    Size=0 if key[-1] == '/' else stat.st_size,
                    LastModified=last_modified or mtime_to_str(stat.st_mtime),
                    ETag=str_to_etag(key if key_to_etag else path),
                    StorageClass=storage_class)
        contents_list.append(item)

    list_bucket_result = dict(Name=name,
                              Prefix=prefix,
                              StartAfter=start_after,
                              MaxKeys=max_keys,
                              Delimiter=delimiter,
                              IsTruncated=next_continuation_token is not None,
                              ContinuationToken=continuation_token)
    if next_continuation_token is not None:
        list_bucket_result.update(NextContinuationToken=next_continuation_token)
    if contents_list:
        list_bucket_result.update(Contents=contents_list)
    if common_prefixes_list:
        list_bucket_result.update(CommonPrefixes=[dict(Prefix=prefix) for prefix in common_prefixes_list])
    return list_bucket_result


def list_s3_bucket_v1(bucket_mapping: Dict[str, str],
                      name: str = None,
                      delimiter: str = None,
                      prefix: str = None,
                      max_keys: int = None,
                      marker: str = None,
                      storage_class: str = None,
                      last_modified: str = None,
                      key_to_etag: bool = False) -> Dict:
    """
    Implements AWS GET Bucket (List Objects) Version 1
    (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html)
    for the local filesystem.

    :param bucket_mapping: mapping from the bucket's top-level names to directory paths in the local file system
    :param name: The bucket name, defaults to "s3bucket"
    :param delimiter: A delimiter is a character you use to group keys.
           If you specify a prefix, all of the keys that contain the same string
           between the prefix and the first occurrence of the delimiter after
           the prefix are grouped under a single result element called CommonPrefixes.
           If you don't specify the prefix parameter, the substring starts at the
           beginning of the key.
           The keys that are grouped under the CommonPrefixes result element are not
           returned elsewhere in the response.
           Refer to AWS docs for details. No default.
    :param prefix: Limits the response to keys that begin with the specified prefix.
           You can use prefixes to separate a bucket into different groupings of keys.
           (You can think of using prefix to make groups in the same way you'd use
           a folder in a file system.)
           Refer to AWS docs for details. No default.
    :param max_keys: Sets the maximum number of keys returned in the response body.
           If you want to retrieve fewer than the default 1000 keys,
           you can add this to your request.
           The response might contain fewer keys, but it never contains more.
           If there are additional keys that satisfy the search criteria, but these
           keys were not returned because max-keys was exceeded, the response
           contains <IsTruncated>true</IsTruncated>.
           To return the additional keys, see NextMarker.
           Refer to AWS docs for details.
    :param marker: Indicates the object key to start with when listing objects in a bucket.
           All objects are listed in the dictionary order.
           Refer to AWS docs for details. No default.
    :param storage_class: Refer to AWS docs for details. Defaults to "STANDARD".
    :param last_modified: For testing only: always use this value for the "LastModified" entry of results
    :param key_to_etag: For testing only: use key to produce MD5 hashes for the "ETag" entry of results
    :return: A dictionary that represents the contents of a "ListBucketResult". Refer to AWS docs for details.
    """
    name = name or 's3bucket'
    max_keys = max_keys or 1000
    storage_class = storage_class or 'STANDARD'

    contents_list = []
    is_truncated = False
    next_marker = None
    marker_seen = marker is None
    common_prefixes_list = []
    common_prefixes_set = set()

    for key, path in list_s3_bucket_keys(bucket_mapping):

        if len(contents_list) == max_keys:
            is_truncated = True
            next_marker = key
            break

        if key == marker:
            marker_seen = True

        if not marker_seen:
            continue

        if prefix and not key.startswith(prefix):
            continue

        if delimiter:
            index = key.find(delimiter, len(prefix) if prefix else 0)
            if index >= 0:
                key = key[:index + len(delimiter)]
                if key not in common_prefixes_set:
                    common_prefixes_set.add(key)
                    common_prefixes_list.append(key)
                continue

        stat = os.stat(path)
        item = dict(Key=key,
                    Size=0 if key[-1] == '/' else stat.st_size,
                    LastModified=last_modified or mtime_to_str(stat.st_mtime),
                    ETag=str_to_etag(key if key_to_etag else path),
                    StorageClass=storage_class)
        contents_list.append(item)

    list_bucket_result = dict(Name=name,
                              Prefix=prefix,
                              Marker=marker,
                              MaxKeys=max_keys,
                              Delimiter=delimiter,
                              IsTruncated=is_truncated)
    if is_truncated:
        list_bucket_result.update(NextMarker=next_marker)
    if contents_list:
        list_bucket_result.update(Contents=contents_list)
    if common_prefixes_list:
        list_bucket_result.update(CommonPrefixes=[dict(Prefix=prefix) for prefix in common_prefixes_list])
    return list_bucket_result


def list_s3_bucket_keys(bucket_mapping: Dict[str, str]) -> Iterator[Tuple[str, str]]:
    bucket_entry_keys = sorted(list(bucket_mapping.keys()))

    for bucket_entry_key in bucket_entry_keys:

        bucket_entry_path = bucket_mapping[bucket_entry_key]
        if not os.path.isdir(bucket_entry_path):
            raise ValueError(f'Value for key {bucket_entry_key!r} is not a directory: {bucket_entry_path}')

        yield bucket_entry_key + '/', bucket_entry_path

        for root, dirs, files in os.walk(bucket_entry_path):

            dirs.sort()
            files.sort()

            sub_key = root[len(bucket_entry_path):].replace('\\', '/') + '/'
            if sub_key[0] == '/':
                sub_key = sub_key[1:]

            if sub_key:
                yield bucket_entry_key + '/' + sub_key, root

            for file in files:
                yield bucket_entry_key + '/' + sub_key + file, os.path.join(root, file)


def list_bucket_result_to_xml(list_bucket_result):
    return dict_to_xml('ListBucketResult',
                       list_bucket_result,
                       root_element_attrs=dict(xmlns="http://s3.amazonaws.com/doc/2006-03-01/"))


def dict_to_xml(root_element_name: str, content_dict: Dict, root_element_attrs: Dict = None) -> str:
    lines = []
    _value_to_xml(lines, root_element_name, content_dict, element_attrs=root_element_attrs)
    return '\n'.join(lines)


def _value_to_xml(lines: List[str],
                  element_name: str,
                  element_value: Any,
                  element_attrs: Dict = None,
                  indent: int = 0):
    attrs = ''
    if element_attrs:
        for attr_name, attr_value in element_attrs.items():
            attrs += f' {attr_name}=\"{_value_to_text(attr_value)}\"'
    if element_value is None:
        lines.append(f'{indent * "  "}<{element_name}{attrs}/>')
    elif isinstance(element_value, dict):
        lines.append(f'{indent * "  "}<{element_name}{attrs}>')
        for sub_element_name, sub_element_value in element_value.items():
            _value_to_xml(lines, sub_element_name, sub_element_value, indent=indent + 1)
        lines.append(f'{indent * "  "}</{element_name}>')
    elif isinstance(element_value, list):
        for item in element_value:
            _value_to_xml(lines, element_name, item, indent=indent)
    else:
        lines.append(f'{indent * "  "}<{element_name}{attrs}>{_value_to_text(element_value)}</{element_name}>')


def _value_to_text(value) -> str:
    if isinstance(value, bool):
        return 'true' if value else 'false'
    return str(value)


def mtime_to_str(mtime) -> str:
    return str(datetime.datetime.fromtimestamp(mtime))


def str_to_etag(s) -> str:
    return '"' + hashlib.md5(bytes(s, encoding='utf-8')).hexdigest() + '"'


# xcube/webapi/service.py

# The MIT License (MIT)
# Copyright (c) 2021 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import asyncio
import configparser
import json
import os
import os.path
import signal
import sys
import time
import traceback
from datetime import datetime
from json import JSONDecodeError
from typing import Optional, Dict, List, Tuple, Mapping

import tornado.escape
import tornado.options
from tornado.ioloop import IOLoop
from tornado.log import enable_pretty_logging
from tornado.web import Application
from tornado.web import RequestHandler

from xcube.constants import LOG
from xcube.core.dsio import is_s3_url
from xcube.core.mldataset import guess_ml_dataset_format
from xcube.util.cache import parse_mem_size
from xcube.util.caseless import caseless_dict
from xcube.util.config import load_configs
from xcube.util.config import load_json_or_yaml_config
from xcube.util.undefined import UNDEFINED
from xcube.version import version
from xcube.webapi.context import ServiceContext
from xcube.webapi.defaults import DEFAULT_ADDRESS
from xcube.webapi.defaults import DEFAULT_LOG_PREFIX
from xcube.webapi.defaults import DEFAULT_PORT
from xcube.webapi.defaults import DEFAULT_TILE_CACHE_SIZE
from xcube.webapi.defaults import DEFAULT_TILE_COMP_MODE
from xcube.webapi.defaults import DEFAULT_TRACE_PERF
from xcube.webapi.defaults import DEFAULT_UPDATE_PERIOD
from xcube.webapi.errors import ServiceBadRequestError
from xcube.webapi.reqparams import RequestParams

__author__ = "Norman Fomferra (Brockmann Consult GmbH)"

SNAP_CPD_LIST = list()


class Service:
    """
    A web service that provides a remote API to some application.
    """

    def __init__(self,
                 application: Application,
                 prefix: str = None,
                 address: str = DEFAULT_ADDRESS,
                 port: int = DEFAULT_PORT,
                 cube_paths: List[str] = None,
                 styles: Dict[str, Tuple] = None,
                 config_file: Optional[str] = None,
                 base_dir: Optional[str] = None,
                 tile_cache_size: Optional[str] = DEFAULT_TILE_CACHE_SIZE,
                 tile_comp_mode: int = DEFAULT_TILE_COMP_MODE,
                 update_period: Optional[float] = DEFAULT_UPDATE_PERIOD,
                 trace_perf: bool = DEFAULT_TRACE_PERF,
                 log_file_prefix: str = DEFAULT_LOG_PREFIX,
                 log_to_stderr: bool = False,
                 aws_prof: str = None,
                 aws_env: bool = False) -> None:

        """
        Start a tile service.

        The *service_info_file*, if given, represents the service in the filesystem, similar to
        the ``/var/run/`` directory on Linux systems.

        If the service file exist and its information is compatible with the requested *port*, *address*, *caller*, then
        this function simply returns without taking any other actions.

        :param application: The Tornado web application
        :param address: the address
        :param port: the port number
        :param cube_paths: optional list of cube paths
        :param config_file: optional configuration file
        :param base_dir: optional base directory
        :param update_period: if not-None, time of idleness in seconds before service is updated
        :param log_file_prefix: Log file prefix, default is "xcube-serve.log"
        :param log_to_stderr: Whether logging should be shown on stderr
        :return: service information dictionary
        """
        if config_file and cube_paths:
            raise ValueError("config_file and cube_paths cannot be given both")
        if config_file and styles:
            raise ValueError("config_file and styles cannot be given both")
        if config_file and aws_prof:
            raise ValueError("config_file and aws_profile cannot be given both")
        if config_file and aws_env:
            raise ValueError("config_file and aws_env cannot be given both")

        global SNAP_CPD_LIST
        if config_file:
            SNAP_CPD_LIST = _get_custom_color_list(config_file)

        log_dir = os.path.dirname(log_file_prefix)
        if log_dir and not os.path.isdir(log_dir):
            os.makedirs(log_dir, exist_ok=True)

        options = tornado.options.options
        options.log_file_prefix = log_file_prefix or DEFAULT_LOG_PREFIX
        options.log_to_stderr = log_to_stderr
        enable_pretty_logging()

        tile_cache_capacity = parse_mem_size(tile_cache_size)

        config = None
        if cube_paths:
            config = new_default_config(cube_paths, styles, aws_prof=aws_prof, aws_env=aws_env)

        self.config_file = os.path.abspath(config_file) if config_file else None
        self.update_period = update_period
        self.update_timer = None
        self.config_error = None
        self.service_info = dict(port=port,
                                 address=address,
                                 started=datetime.now().isoformat(sep=' '),
                                 pid=os.getpid())

        self.context = ServiceContext(prefix=prefix,
                                      config=config,
                                      base_dir=base_dir,
                                      trace_perf=trace_perf,
                                      tile_comp_mode=tile_comp_mode,
                                      tile_cache_capacity=tile_cache_capacity)
        self._maybe_load_config()

        application.service_context = self.context
        application.time_of_last_activity = time.process_time()
        self.application = application

        # Register handlers for common termination signals
        signal.signal(signal.SIGINT, self._sig_handler)
        signal.signal(signal.SIGTERM, self._sig_handler)

        self.server = application.listen(port, address=address or 'localhost')
        # Ensure we have the same event loop in all threads
        asyncio.set_event_loop_policy(_GlobalEventLoopPolicy(asyncio.get_event_loop()))
        self._maybe_load_config()
        self._maybe_install_update_check()
        self._shutdown_requested = False

    def start(self):
        address = self.service_info['address']
        port = self.service_info['port']
        test_url = self.context.get_service_url(f"http://{address}:{port}", "datasets")
        LOG.info(f'service running, listening on {address}:{port}, try {test_url}')
        LOG.info(f'press CTRL+C to stop service')
        if not self.context.config.get('Datasets', []) \
                and not self.context.config.get('DataStores', []):
            LOG.warning('no datasets or data stores configured')
        tornado.ioloop.PeriodicCallback(self._try_shutdown, 100).start()
        IOLoop.current().start()

    def stop(self, kill=False):
        """
        Stops the Tornado web server.
        """
        if kill:
            sys.exit(0)
        else:
            IOLoop.current().add_callback(self._on_shutdown)

    def _on_shutdown(self):

        LOG.info('stopping service...')

        # noinspection PyUnresolvedReferences,PyBroadException
        try:
            self.update_timer.cancel()
        except Exception:
            pass

        if self.server:
            self.server.stop()
            self.server = None

        IOLoop.current().stop()
        LOG.info('service stopped.')

    def _try_shutdown(self):
        if self._shutdown_requested:
            self._on_shutdown()

    # noinspection PyUnusedLocal
    def _sig_handler(self, sig, frame):
        LOG.warning(f'caught signal {sig}')
        self._shutdown_requested = True

    def _maybe_install_update_check(self):
        if self.config_file is None or self.update_period is None or self.update_period <= 0:
            return
        IOLoop.current().call_later(self.update_period, self._maybe_check_for_updates)

    def _maybe_check_for_updates(self):
        self._maybe_load_config()
        self._maybe_install_update_check()

    def _maybe_load_config(self):
        config_file = self.config_file
        if config_file is None:
            return

        try:
            stat = os.stat(config_file)
        except OSError as e:
            if self.config_error is None:
                LOG.error(f'configuration file {config_file!r}: {e}')
                self.config_error = e
            return

        if self.context.config_mtime != stat.st_mtime:
            self.context.config_mtime = stat.st_mtime
            try:
                self.context.config = load_json_or_yaml_config(config_file)
                self.config_error = None
                LOG.info(f'configuration file {config_file!r} successfully loaded')
            except ValueError as e:
                if self.config_error is None:
                    LOG.error(f'configuration file {config_file!r}: {e}')
                    self.config_error = e


# noinspection PyAbstractClass
class ServiceRequestHandler(RequestHandler):

    def __init__(self, application, request, **kwargs):
        super().__init__(application, request, **kwargs)
        self._params = ServiceRequestParams(self)

    def set_caseless_query_arguments(self):
        self.request.query_arguments = caseless_dict(self.request.query_arguments or {})

    @property
    def service_context(self) -> ServiceContext:
        # noinspection PyUnresolvedReferences
        return self.application.service_context

    @property
    def base_url(self):
        return self.request.protocol + '://' + self.request.host

    @property
    def params(self) -> 'ServiceRequestParams':
        return self._params

    def set_default_headers(self):
        self.set_header('Server', f'xcube-server/{version}')
        self.set_header('Access-Control-Allow-Origin', '*')
        self.set_header('Access-Control-Allow-Methods',
                        'GET,PUT,DELETE,OPTIONS')
        self.set_header('Access-Control-Allow-Headers',
                        'x-requested-with,access-control-allow-origin,'
                        'authorization,content-type')

    # noinspection PyUnusedLocal
    def options(self, *args, **kwargs):
        self.set_status(204)
        self.finish()

    def get_body_as_json_object(self, name="JSON object"):
        """ Get the body argument as JSON object. """
        try:
            return tornado.escape.json_decode(self.request.body)
        except (JSONDecodeError, TypeError, ValueError) as e:
            raise ServiceBadRequestError(f"Invalid or missing {name} in request body") from e

    def on_finish(self):
        """
        Store time of last activity so we can measure time of inactivity and then optionally auto-exit.
        """
        self.application.time_of_last_activity = time.process_time()

    def write_error(self, status_code, **kwargs):
        self.set_header('Content-Type', 'application/json')
        # if self.settings.get("serve_traceback") and "exc_info" in kwargs:
        if "exc_info" in kwargs:
            # in debug mode, try to send a traceback
            lines = []
            for line in traceback.format_exception(*kwargs["exc_info"]):
                lines.append(line)
            self.finish(json.dumps({
                'error': {
                    'code': status_code,
                    'message': self._reason,
                    'traceback': lines,
                }
            }, indent=2))
        else:
            self.finish(json.dumps({
                'error': {
                    'code': status_code,
                    'message': self._reason,
                }
            }, indent=2))


class ServiceRequestParams(RequestParams):
    def __init__(self, handler: RequestHandler):
        self.handler = handler

    def get_query_arguments(self) -> Mapping[str, str]:
        handler = self.handler
        request = handler.request
        return {name: handler.get_query_argument(name) for name in request.query_arguments.keys()}

    def get_query_argument(self, name: str, default: Optional[str] = UNDEFINED) -> Optional[str]:
        """
        Get query argument.
        :param name: Query argument name
        :param default: Default value.
        :return: the value or none
        :raise: ServiceBadRequestError
        """
        value = self.handler.get_query_argument(name, default=default)
        if value == UNDEFINED:
            raise ServiceBadRequestError(f'Missing query parameter "{name}"')
        return value


# noinspection PyAbstractClass
class _GlobalEventLoopPolicy(asyncio.DefaultEventLoopPolicy):
    """
    Event loop policy that has one fixed global loop for all threads.

    We use it for the following reason: As of Tornado 5 IOLoop.current() no longer has
    a single global instance. It is a thread-local instance, but only on the main thread.
    Other threads have no IOLoop instance by default.

    _GlobalEventLoopPolicy is a fix that allows us to access the same IOLoop
    in all threads.

    Usage::

        asyncio.set_event_loop_policy(_GlobalEventLoopPolicy(asyncio.get_event_loop()))

    """

    def __init__(self, global_loop):
        super().__init__()
        self._global_loop = global_loop

    def get_event_loop(self):
        return self._global_loop


def url_pattern(pattern: str):
    """
    Convert a string *pattern* where any occurrences of ``{{NAME}}`` are replaced by an equivalent
    regex expression which will assign matching character groups to NAME. Characters match until
    one of the RFC 2396 reserved characters is found or the end of the *pattern* is reached.

    The function can be used to map URLs patterns to request handlers as desired by the Tornado web server, see
    http://www.tornadoweb.org/en/stable/web.html

    RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
    the following reserved characters::

        reserved    = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","

    :param pattern: URL pattern
    :return: equivalent regex pattern
    :raise ValueError: if *pattern* is invalid
    """
    name_pattern = r'(?P<%s>[^\;\/\?\:\@\&\=\+\$\,]+)'
    reg_expr = ''
    pos = 0
    while True:
        pos1 = pattern.find('{{', pos)
        if pos1 >= 0:
            pos2 = pattern.find('}}', pos1 + 2)
            if pos2 > pos1:
                name = pattern[pos1 + 2:pos2]
                if not name.isidentifier():
                    raise ValueError('name in {{name}} must be a valid identifier, but got "%s"' % name)
                reg_expr += pattern[pos:pos1] + (name_pattern % name)
                pos = pos2 + 2
            else:
                raise ValueError('no matching "}}" after "{{" in "%s"' % pattern)

        else:
            reg_expr += pattern[pos:]
            break
    return reg_expr


def new_default_config(cube_paths: List[str],
                       styles: Dict[str, Tuple] = None,
                       aws_prof: str = None,
                       aws_env: bool = False):
    aws_access_key_id = None
    aws_secret_access_key = None
    if aws_prof:
        aws_credentials_config = configparser.ConfigParser()
        aws_credentials_config.read(os.path.expanduser(os.path.join('~', '.aws', 'credentials')))
        aws_access_key_id = aws_credentials_config.get(aws_prof, 'aws_access_key_id', fallback=None)
        aws_secret_access_key = aws_credentials_config.get(aws_prof, 'aws_secret_access_key', fallback=None)
        if aws_access_key_id is None:
            raise ValueError(f'missing aws_access_key_id in AWS credentials')
        if aws_secret_access_key is None:
            raise ValueError(f'missing aws_secret_access_key in AWS credentials')
    if aws_env:
        aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', aws_access_key_id)
        aws_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY', aws_secret_access_key)
        if aws_access_key_id is None:
            raise ValueError(f'environment variable AWS_ACCESS_KEY_ID not set')
        if aws_secret_access_key is None:
            raise ValueError(f'environment variable AWS_SECRET_ACCESS_KEY not set')

    dataset_configs = list()
    index = 0
    for cube_path in cube_paths:
        dataset_config = dict(Identifier=f"dataset_{index + 1}",
                              Format=guess_ml_dataset_format(cube_path),
                              Path=cube_path)
        if is_s3_url(cube_path):
            dataset_config.update(Title=cube_path.split('/')[-1],
                                  FileSystem='s3')
            if aws_access_key_id and aws_secret_access_key:
                dataset_config.update(AccessKeyId=aws_access_key_id,
                                      SecretAccessKey=aws_secret_access_key)
        else:
            dataset_config.update(Title=os.path.split(cube_path)[-1],
                                  FileSystem='file')
        dataset_configs.append(dataset_config)
        index += 1

    config = dict(Datasets=dataset_configs)
    if styles:
        color_mappings = {}
        for var_name, style_data in styles.items():
            try:
                value_min, value_max, color_bar_name = style_data
                style = dict(ValueRange=[value_min, value_max], ColorBar=color_bar_name)
            except (TypeError, ValueError):
                try:
                    value_min, value_max = style_data
                    style = dict(ValueRange=[value_min, value_max])
                except (TypeError, ValueError):
                    raise ValueError(f"illegal style: {var_name}={style_data!r}")
            color_mappings[var_name] = style
        config["Styles"] = [dict(Identifier="default", ColorMappings=color_mappings)]
    return config


# TODO (forman): fix this hack
def _get_custom_color_list(config_file):
    # global: too bad :(
    global SNAP_CPD_LIST
    config = load_configs(config_file) if config_file else {}
    styles = config.get('Styles')
    if isinstance(styles, list):
        for style in styles:
            cm = style.get('ColorMappings')
            if isinstance(cm, dict):
                for key in cm.keys():
                    if 'ColorFile' in cm[key]:
                        cf = cm[key]['ColorFile']
                        if cf not in SNAP_CPD_LIST:
                            SNAP_CPD_LIST.append(cf)
    return SNAP_CPD_LIST


# test/cli/compute-scripts/with-init-and-finalize.py

# noinspection PyUnusedLocal
def compute(variable_a, variable_b, input_params=None, **kwargs):
    a = input_params.get('a', 0.5)
    b = input_params.get('b', 0.5)
    return a * variable_a + b * variable_b


def initialize(input_cubes, input_var_names, input_params):
    if len(input_cubes) != 1:
        raise ValueError("Expected a single input cube")

    if input_var_names and len(input_var_names) != 2:
        raise ValueError("Two variables expected")

    input_cube = input_cubes[0]

    if not input_var_names:
        if 'precipitation' not in input_cube:
            raise ValueError("Cube must have 'precipitation'")
        if 'soil_moisture' not in input_cube:
            raise ValueError("Cube must have 'soil_moisture'")
        input_var_names = ['precipitation', 'soil_moisture']

    illegal_param_names = set(input_params.keys()).difference({"a", "b"})
    if illegal_param_names:
        raise ValueError(f"Illegal parameter(s): {illegal_param_names}")

    return input_var_names, dict(a=input_params.get('a', 0.2), b=input_params.get('b', 0.3))


def finalize(output_cube):
    output_cube.output.attrs.update(units='mg/m^3')
    output_cube.attrs.update(comment='I has a bucket')
    return output_cube


# test/cli/compute-scripts/without-init.py

# noinspection PyUnusedLocal
def compute(variable_a, variable_b, input_params=None, **kwargs):
    a = input_params.get('a', 0.5)
    b = input_params.get('b', 0.5)
    return a * variable_a + b * variable_b


# test/core/byoa/test_config.py

import os.path
import unittest

from xcube.core.byoa import CodeConfig
from xcube.core.byoa import FileSet

INLINE_CODE = (
    'def process_dataset(ds, text="Hello"):\n'
    '    return ds.assign_attrs(comment=f"xcube says {text}")\n'
)

PARENT_DIR = os.path.dirname(__file__)
LOCAL_MODULE_DIR = os.path.join(PARENT_DIR,
                                'test_data', 'user_code')
LOCAL_MODULE_ZIP = os.path.join(PARENT_DIR,
                                'test_data', 'user_code.zip')


class CodeConfigTest(unittest.TestCase):

    def test_from_code_string(self):
        code_config = CodeConfig.from_code(
            INLINE_CODE,
            module_name='user_code_1',
            callable_params={'text': 'good bye'}
        )
        self.assertIsInstance(code_config, CodeConfig)
        self.assertEqual(INLINE_CODE, code_config.inline_code)
        self.assertEqual({'text': 'good bye'}, code_config.callable_params)
        self.assertEqual('user_code_1:process_dataset',
                         code_config.callable_ref)
        self.assertTrue(callable(code_config.get_callable()))

    def test_from_code_with_function_refs(self):
        code_config = CodeConfig.from_code(
            'import xarray as xr\n',
            modify_dataset,
            transform_dataset,
            module_name='user_code_2',
            callable_params={'text': 'good bye'}
        )
        self.assertIsInstance(code_config, CodeConfig)
        self.assertEqual(
            (
                'import xarray as xr\n'
                '\n'
                '\n'
                'def modify_dataset(ds, text="Hello"):\n'
                '    return transform_dataset(ds, text)\n'
                '\n'
                '\n'
                'def transform_dataset(ds, text):\n'
                '    return ds.assign_attrs(comment=f"xcube says {text}")\n'
            ),
            code_config.inline_code)
        self.assertIsNone(code_config.file_set)
        self.assertEqual({'text': 'good bye'}, code_config.callable_params)
        self.assertEqual('user_code_2:modify_dataset', code_config.callable_ref)
        self.assertTrue(callable(code_config.get_callable()))

    def test_from_callable(self):
        code_config = CodeConfig.from_callable(
            modify_dataset,
            callable_params=dict(text='Good bye!')
        )
        self.assertIsInstance(code_config, CodeConfig)
        self.assertIsNone(code_config.inline_code)
        self.assertIsNone(code_config.file_set, FileSet)
        self.assertIsNone(code_config.callable_ref)
        self.assertIs(modify_dataset, code_config.get_callable())

    def test_from_file_set_dir(self):
        code_config = CodeConfig.from_file_set(
            LOCAL_MODULE_DIR,
            'processor:process_dataset'
        )
        self.assertIsInstance(code_config, CodeConfig)
        self.assertIsNone(code_config.inline_code)
        self.assertIsInstance(code_config.file_set, FileSet)
        self.assertEqual('processor:process_dataset', code_config.callable_ref)
        self.assertTrue(callable(code_config.get_callable()))

    def test_from_file_set_zip(self):
        code_config = CodeConfig.from_file_set(
            LOCAL_MODULE_ZIP,
            'processor:process_dataset'
        )
        self.assertIsInstance(code_config, CodeConfig)
        self.assertIsNone(code_config.inline_code)
        self.assertIsInstance(code_config.file_set, FileSet)
        self.assertEqual('processor:process_dataset',
                         code_config.callable_ref)
        self.assertTrue(callable(code_config.get_callable()))

    def test_from_github_release(self):
        code_config = CodeConfig.from_github_archive(
            'dcs4cop',
            'xcube-byoa-examples',
            'v0.1.0.dev0',
            '0.1.0.dev0',
            'xcube_byoa_1.processor:process_dataset'
        )

        self.assertIsInstance(code_config, CodeConfig)
        self.assertIsNone(code_config.inline_code)
        self.assertIsInstance(code_config.file_set, FileSet)
        self.assertEqual('https://github.com/dcs4cop/'
                         'xcube-byoa-examples/archive/v0.1.0.dev0.zip',
                         code_config.file_set.path)
        self.assertEqual('xcube_byoa_1.processor:process_dataset',
                         code_config.callable_ref)

    def test_for_local_from_callable(self):
        user_code_config = CodeConfig.from_callable(modify_dataset)

        local_code_config = user_code_config.for_local()
        self.assertIs(user_code_config, local_code_config)
        # CodeConfigs from for_local() shall be able to load callable
        self.assertIs(modify_dataset, local_code_config.get_callable())

    def test_for_local_from_inline_code(self):
        user_code_config = CodeConfig.from_code(INLINE_CODE)

        local_code_config = user_code_config.for_local()
        self.assertIsInstance(local_code_config, CodeConfig)
        self.assertIsInstance(local_code_config.callable_ref, str)
        self.assertIsNone(local_code_config._callable)
        self.assertIsNone(local_code_config.inline_code)
        self.assertIsInstance(local_code_config.file_set, FileSet)
        self.assertTrue(local_code_config.file_set.is_local_dir())
        self.assertRegex(os.path.basename(local_code_config.file_set.path),
                         'xcube-byoa-*.')
        self.assertRegex(local_code_config.callable_ref,
                         'xcube_byoa_*.:process_dataset')
        # CodeConfigs from for_local() shall be able to load callable
        self.assertTrue(callable(local_code_config.get_callable()))

    def test_for_local_from_file_set_dir(self):
        user_code_config = CodeConfig.from_file_set(
            LOCAL_MODULE_DIR,
            'processor:process_dataset'
        )

        local_code_config = user_code_config.for_local()
        self.assertIs(user_code_config, local_code_config)
        self.assertIsInstance(local_code_config.file_set, FileSet)
        self.assertIs(user_code_config.file_set, local_code_config.file_set)
        self.assertEqual(LOCAL_MODULE_DIR, local_code_config.file_set.path)
        # CodeConfigs from for_local() shall be able to load callable
        self.assertTrue(callable(local_code_config.get_callable()))

    def test_for_local_from_file_set_zip(self):
        user_code_config = CodeConfig.from_file_set(
            LOCAL_MODULE_ZIP,
            'processor:process_dataset'
        )

        local_code_config = user_code_config.for_local()
        self.assertIsInstance(local_code_config, CodeConfig)
        self.assertIsInstance(local_code_config.callable_ref, str)
        self.assertIsNone(local_code_config._callable)
        self.assertIsNone(local_code_config.inline_code)
        self.assertIsInstance(local_code_config.file_set, FileSet)
        self.assertTrue(local_code_config.file_set.is_local_dir())
        self.assertRegex(os.path.basename(local_code_config.file_set.path),
                         'xcube-byoa-*.')
        self.assertEqual('processor:process_dataset',
                         local_code_config.callable_ref)
        # CodeConfigs from for_local() shall be able to load callable
        self.assertTrue(callable(local_code_config.get_callable()))

    def test_for_local_from_github_release(self):
        user_code_config = CodeConfig.from_github_archive(
            gh_org='dcs4cop',
            gh_repo='xcube-byoa-examples',
            gh_tag='v0.1.0.dev0',
            gh_release='0.1.0.dev0',
            callable_ref='xcube_byoa_ex1.processor:process_dataset',
        )

        local_code_config = user_code_config.for_local()
        self.assertIsInstance(local_code_config, CodeConfig)
        self.assertIsInstance(local_code_config.callable_ref, str)
        self.assertIsNone(local_code_config._callable)
        self.assertIsNone(local_code_config.inline_code)
        self.assertIsInstance(local_code_config.file_set, FileSet)
        self.assertTrue(local_code_config.file_set.is_local_dir())
        self.assertRegex(os.path.basename(local_code_config.file_set.path),
                         'xcube-byoa-*.')
        self.assertEqual('xcube_byoa_ex1.processor:process_dataset',
                         local_code_config.callable_ref)
        # CodeConfigs from for_local() shall be able to load callable
        self.assertTrue(callable(local_code_config.get_callable()))

    def test_for_local_illegal_state(self):
        code_config = CodeConfig.from_code(INLINE_CODE)
        code_config.inline_code = None
        with self.assertRaises(RuntimeError) as e:
            code_config.for_local()
        self.assertEqual(
            ('for_local() failed due to an invalid CodeConfig state',),
            e.exception.args
        )

    def test_for_service_from_callable(self):
        user_code_config = CodeConfig.from_callable(transform_dataset)

        service_code_config = user_code_config.for_service()
        self.assertIsInstance(service_code_config, CodeConfig)
        self.assertIsInstance(service_code_config.callable_ref, str)
        self.assertIsNone(service_code_config._callable)
        self.assertIsNone(service_code_config.inline_code)
        self.assertIsInstance(service_code_config.file_set, FileSet)
        self.assertTrue(service_code_config.file_set.is_local_zip())

    def test_for_service_from_file_set_zip(self):
        user_code_config = CodeConfig.from_file_set(
            LOCAL_MODULE_ZIP,
            'processor:process_dataset'
        )

        service_code_config = user_code_config.for_service()
        self.assertIs(user_code_config, service_code_config)
        self.assertIs(user_code_config.file_set, service_code_config.file_set)
        self.assertIsInstance(service_code_config.file_set, FileSet)
        self.assertEqual(LOCAL_MODULE_ZIP, service_code_config.file_set.path)

    def test_for_service_from_file_set_dir(self):
        user_code_config = CodeConfig.from_file_set(
            LOCAL_MODULE_DIR,
            'processor:process_dataset'
        )

        service_code_config = user_code_config.for_service()
        self.assertIsInstance(service_code_config, CodeConfig)
        self.assertIsInstance(service_code_config.callable_ref, str)
        self.assertIsNone(service_code_config._callable)
        self.assertIsNone(service_code_config.inline_code)
        self.assertIsInstance(service_code_config.file_set, FileSet)
        self.assertTrue(service_code_config.file_set.is_local_zip())
        self.assertRegex(os.path.basename(service_code_config.file_set.path),
                         'xcube-byoa-*.')
        self.assertEqual('processor:process_dataset',
                         service_code_config.callable_ref)

    def test_for_service_from_github_release(self):
        user_code_config = CodeConfig.from_github_archive(
            gh_org='dcs4cop',
            gh_repo='xcube-byoa-examples',
            gh_tag='v0.1.0.dev0',
            gh_release='0.1.0.dev0',
            callable_ref='xcube_byoa_1.processor:process_dataset'
        )

        service_code_config = user_code_config.for_service()
        self.assertIsInstance(service_code_config, CodeConfig)
        self.assertIsInstance(service_code_config.callable_ref, str)
        self.assertIsNone(service_code_config._callable)
        self.assertIsNone(service_code_config.inline_code)
        self.assertIsInstance(service_code_config.file_set, FileSet)
        self.assertFalse(service_code_config.file_set.is_local_dir())
        self.assertEqual('https://github.com/dcs4cop/'
                         'xcube-byoa-examples/archive/v0.1.0.dev0.zip',
                         service_code_config.file_set.path)
        self.assertEqual('xcube_byoa_1.processor:process_dataset',
                         service_code_config.callable_ref)

    def test_for_service_illegal_state(self):
        code_config = CodeConfig.from_code(INLINE_CODE)
        code_config.inline_code = None
        with self.assertRaises(RuntimeError) as e:
            code_config.for_service()
        self.assertEqual(
            ('for_service() failed due to an invalid CodeConfig state',),
            e.exception.args
        )

    def test_to_dict(self):
        d = CodeConfig.from_callable(modify_dataset).to_dict()
        # Shall we raise instead?
        self.assertEqual({}, d)

        d = CodeConfig.from_code(
            INLINE_CODE, module_name='user_code'
        ).to_dict()
        self.assertEqual(
            {
                'callable_ref': 'user_code:process_dataset',
                'inline_code': INLINE_CODE,
            },
            d
        )

        d = CodeConfig.from_file_set(
            FileSet('github://dcs4cop:xcube@v0.8.2.dev0',
                    includes='*.py'),
            callable_ref=('test.core.byoa.test_config:'
                          'modify_dataset')
        ).to_dict()
        self.assertIsInstance(d.get('file_set'), dict)
        self.assertEqual(('test.core.byoa.test_config:'
                          'modify_dataset'),
                         d.get('callable_ref'))


def modify_dataset(ds, text="Hello"):
    return transform_dataset(ds, text)


def transform_dataset(ds, text):
    return ds.assign_attrs(comment=f"xcube says {text}")


# test/core/byoa/test_fileset.py

import os
import os.path
import os.path
import unittest

from xcube.core.byoa.constants import TEMP_FILE_PREFIX
from xcube.core.byoa import FileSet

PARENT_DIR = os.path.dirname(__file__)


class FileSetTest(unittest.TestCase):

    def test_is_local(self):
        self.assertTrue(FileSet('test_data/user_code').is_local())
        self.assertTrue(FileSet('file://test_data/user_code').is_local())
        self.assertFalse(FileSet('s3://xcube/user_code').is_local())
        self.assertFalse(FileSet('https://xcube/user_code.zip').is_local())
        self.assertFalse(FileSet('github://dcs4cop:xcube@v0.8.1').is_local())

    def test_is_local_dir(self):
        local_dir_path = os.path.join(PARENT_DIR, 'test_data', 'user_code')
        self.assertTrue(FileSet(local_dir_path).is_local_dir())
        self.assertTrue(FileSet('file://' + local_dir_path).is_local_dir())
        self.assertFalse(FileSet('s3://eurodatacube/test/').is_local_dir())
        self.assertFalse(FileSet('s3://eurodatacube/test.zip').is_local_dir())

    def test_is_local_zip(self):
        local_zip_path = os.path.join(PARENT_DIR, 'test_data', 'user_code.zip')
        self.assertTrue(FileSet(local_zip_path).is_local_zip())
        self.assertTrue(FileSet('file://' + local_zip_path).is_local_zip())
        self.assertFalse(FileSet('s3://eurodatacube/test/').is_local_zip())
        self.assertFalse(FileSet('s3://eurodatacube/test.zip').is_local_zip())

    def test_keys(self):
        self._test_keys_for_local('test_data/user_code')
        self._test_keys_for_local('test_data/user_code.zip')

    def _test_keys_for_local(self, rel_path: str):
        path = os.path.join(PARENT_DIR, rel_path)

        file_set = FileSet(path, excludes=['__pycache__'])
        self.assertEqual(
            {
                'NOTES.md',
                'processor.py',
                'impl/__init__.py',
                'impl/algorithm.py',
            },
            set(file_set.keys()))

        file_set = FileSet(path, includes=['*.py'])
        self.assertEqual(
            {
                'processor.py',
                'impl/__init__.py',
                'impl/algorithm.py',
            },
            set(file_set.keys()))

        file_set = FileSet(path, excludes=['NOTES.md', '__pycache__'])
        self.assertEqual(
            {
                'processor.py',
                'impl/__init__.py',
                'impl/algorithm.py',
            },
            set(file_set.keys()))

    def test_to_local_zip_then_to_local_dir(self):
        dir_path = os.path.join(PARENT_DIR, 'test_data', 'user_code')

        file_set = FileSet(dir_path, excludes=['__pycache__'])
        zip_file_set = file_set.to_local_zip()
        self.assertIsInstance(zip_file_set, FileSet)
        self.assertTrue(zip_file_set.is_local())
        self.assertFalse(zip_file_set.is_local_dir())
        self.assertTrue(zip_file_set.is_local_zip())
        self.assertRegex(zip_file_set.path,
                         f'^.*{TEMP_FILE_PREFIX}.*\\.zip$')
        self.assertEqual(
            {
                'NOTES.md',
                'processor.py',
                'impl/__init__.py',
                'impl/algorithm.py',
            },
            set(zip_file_set.keys())
        )

        dir_file_set = zip_file_set.to_local_dir()
        self.assertIsInstance(dir_file_set, FileSet)
        self.assertTrue(zip_file_set.is_local())
        self.assertTrue(dir_file_set.is_local_dir())
        self.assertFalse(dir_file_set.is_local_zip())
        self.assertRegex(dir_file_set.path,
                         f'^.*{TEMP_FILE_PREFIX}.*$')
        self.assertEqual(
            {
                'NOTES.md',
                'processor.py',
                'impl/__init__.py',
                'impl/algorithm.py',
            },
            set(dir_file_set.keys())
        )

    def test_to_local_dir_then_to_local_zip(self):
        zip_path = os.path.join(PARENT_DIR, 'test_data', 'user_code.zip')

        dir_file_set = FileSet(zip_path).to_local_dir()
        self.assertIsInstance(dir_file_set, FileSet)
        self.assertTrue(dir_file_set.is_local())
        self.assertTrue(dir_file_set.is_local_dir())
        self.assertFalse(dir_file_set.is_local_zip())
        self.assertRegex(dir_file_set.path,
                         f'^.*{TEMP_FILE_PREFIX}.*$')
        self.assertEqual(
            {
                'NOTES.md',
                'processor.py',
                'impl/__init__.py',
                'impl/algorithm.py',
            },
            set(dir_file_set.keys())
        )

        zip_file_set = dir_file_set.to_local_zip()
        self.assertIsInstance(zip_file_set, FileSet)
        self.assertTrue(dir_file_set.is_local())
        self.assertFalse(zip_file_set.is_local_dir())
        self.assertTrue(zip_file_set.is_local_zip())
        self.assertRegex(zip_file_set.path,
                         f'^.*{TEMP_FILE_PREFIX}.*\\.zip$')
        self.assertEqual(
            {
                'NOTES.md',
                'processor.py',
                'impl/__init__.py',
                'impl/algorithm.py',
            },
            set(zip_file_set.keys())
        )

    def test_to_dict(self):
        file_set = FileSet('test_data/user_code')
        self.assertEqual(
            {'path': 'test_data/user_code'},
            file_set.to_dict())

        file_set = FileSet('test_data/user_code',
                           includes=['*.py'])
        self.assertEqual(
            {
                'path': 'test_data/user_code',
                'includes': ['*.py']
            },
            file_set.to_dict())

        file_set = FileSet('test_data/user_code',
                           excludes=['NOTES.md'])
        self.assertEqual(
            {
                'path': 'test_data/user_code',
                'excludes': ['NOTES.md']
            },
            file_set.to_dict())

        file_set = FileSet('s3://xcube/user_code',
                           sub_path='test',
                           storage_params={'anon': True},
                           includes=['*.py'],
                           excludes=['NOTES.md'])
        self.assertEqual(
            {
                'path': 's3://xcube/user_code',
                'sub_path': 'test',
                'storage_params': {'anon': True},
                'includes': ['*.py'],
                'excludes': ['NOTES.md'],
            },
            file_set.to_dict())


# test/core/gen/helpers.py

import os


def get_inputdata_path(name: str) -> str:
    return os.path.join(os.path.dirname(__file__), 'inputdata', name)


# test/core/gen/test_config.py

import os
import shutil
import unittest

import yaml

from xcube.core.gen.config import get_config_dict

TEMP_PATH_FOR_YAML = './temp_test_data_for_xcube_tests'

CONFIG_1_NAME = 'config_1.yaml'
CONFIG_2_NAME = 'config_2.yaml'
CONFIG_1_FILE_LIST = [(os.path.join(TEMP_PATH_FOR_YAML, CONFIG_1_NAME)),
                      (os.path.join(TEMP_PATH_FOR_YAML, CONFIG_2_NAME))]
CONFIG_1_YAML = """
output_size: [2000, 1000] 
output_region: [0, 20, 20, 30] 
output_variables: 
  - x
  - y
  - z*
"""

CONFIG_3_NAME = 'config_3.yaml'
CONFIG_4_NAME = 'config_4.yaml'
CONFIG_2_FILE_LIST = [(os.path.join(TEMP_PATH_FOR_YAML, CONFIG_3_NAME)),
                      (os.path.join(TEMP_PATH_FOR_YAML, CONFIG_4_NAME))]
CONFIG_2_YAML = """
: output_variables: 
  - x
 6--
"""


def _create_temp_yaml(temp_path_for_yaml, config_file_name, config_yaml):
    if not os.path.exists(TEMP_PATH_FOR_YAML):
        try:
            os.mkdir(os.path.join(temp_path_for_yaml))
        except OSError as e:
            print(e)
            print("Creation of the directory %s failed" % temp_path_for_yaml)
        else:
            print("Successfully created the directory %s " % temp_path_for_yaml)
            yaml_path = os.path.join(temp_path_for_yaml, config_file_name)
            with open(yaml_path, 'w') as outfile:
                yaml.dump(yaml.full_load(config_yaml), outfile)
            return yaml_path

    else:
        yaml_path = os.path.join(temp_path_for_yaml, config_file_name)
        with open(yaml_path, 'w') as outfile:
            yaml.dump(config_yaml, outfile, default_flow_style=False)
        return yaml_path


class GetConfigDictTest(unittest.TestCase):
    def test_config_file_alone(self):
        try:
            _create_temp_yaml(TEMP_PATH_FOR_YAML, CONFIG_2_NAME, CONFIG_1_YAML)
            config_obj = dict(config_files=CONFIG_1_FILE_LIST)
            _create_temp_yaml(TEMP_PATH_FOR_YAML, CONFIG_1_NAME, config_obj)

            config = get_config_dict(**config_obj)
            self.assertIsNotNone(config)
            self.assertEqual([2000, 1000], config['output_size'])
            self.assertEqual([0, 20, 20, 30], config['output_region'])
            self.assertEqual([('x', None), ('y', None), ('z*', None)], config['output_variables'])
        finally:
            if os.path.exists(TEMP_PATH_FOR_YAML):
                shutil.rmtree(TEMP_PATH_FOR_YAML)
                print('Successfully removed folder')

    def test_config_file_overwritten_by_config_obj(self):
        try:
            _create_temp_yaml(TEMP_PATH_FOR_YAML, CONFIG_2_NAME, CONFIG_1_YAML)
            config_obj = dict(config_files=CONFIG_1_FILE_LIST,
                              output_variables='a,b')
            _create_temp_yaml(TEMP_PATH_FOR_YAML, CONFIG_1_NAME, config_obj)
            config = get_config_dict(**config_obj)
            self.assertIn('output_variables', config)
            self.assertIsNotNone(['a', 'b'], config['output_variables'])
        finally:
            if os.path.exists(TEMP_PATH_FOR_YAML):
                shutil.rmtree(TEMP_PATH_FOR_YAML)
                print('Successfully removed folder')

    def test_config_file_does_not_exist(self):
        config_obj = dict(config_files=['bibo.yaml', ])
        with self.assertRaises(ValueError) as cm:
            get_config_dict(**config_obj)
        self.assertEqual("Cannot find configuration 'bibo.yaml'",
                         f'{cm.exception}')

    def test_output_size_option(self):
        config_obj = dict(output_size='120, 140')
        config = get_config_dict(**config_obj)
        self.assertIn('output_size', config)
        self.assertEqual([120, 140], config['output_size'])

        config_obj = dict(output_size='120,abc')
        with self.assertRaises(ValueError) as cm:
            get_config_dict(**config_obj)
        self.assertEqual(
            "output_size must have the form <width>,<height>, where both values must be positive integer numbers",
            f'{cm.exception}')

    def test_output_region_option(self):
        config_obj = dict(output_region='-10.5, 5., 10.5, 25.')
        config = get_config_dict(**config_obj)
        self.assertIn('output_region', config)
        self.assertEqual([-10.5, 5., 10.5, 25.], config['output_region'])

        config_obj = dict(output_region='50,_2,55,21')
        with self.assertRaises(ValueError) as cm:
            get_config_dict(**config_obj)
        self.assertEqual("output_region must have the form <lon_min>,<lat_min>,<lon_max>,<lat_max>,"
                         " where all four numbers must be floating point numbers in degrees",
                         f'{cm.exception}')

        config_obj = dict(output_region='50, 20, 55')
        with self.assertRaises(ValueError) as cm:
            get_config_dict(**config_obj)
        self.assertEqual("output_region must have the form <lon_min>,<lat_min>,<lon_max>,<lat_max>,"
                         " where all four numbers must be floating point numbers in degrees",
                         f'{cm.exception}')

    def test_output_variables_option(self):
        config_obj = dict(output_variables='hanni, nanni, pfanni')
        config = get_config_dict(**config_obj)
        self.assertIn('output_variables', config)
        self.assertEqual([('hanni', None), ('nanni', None), ('pfanni', None)],
                         config['output_variables'])

        config_obj = dict(output_variables='')
        with self.assertRaises(ValueError) as cm:
            get_config_dict(**config_obj)
        self.assertEqual("output_variables must be a list of existing variable names",
                         f'{cm.exception}')

        config_obj = dict(output_variables='a*,')
        with self.assertRaises(ValueError) as cm:
            get_config_dict(**config_obj)
        self.assertEqual("output_variables must be a list of existing variable names",
                         f'{cm.exception}')

    # This test is still not running correcly, needs to be fixed. TODO: AliceBalfanz
    # def test_config_file_with_invalid_yaml(self):
    #     try:
    #         _create_temp_yaml(TEMP_PATH_FOR_YAML, CONFIG_4_NAME, CONFIG_2_YAML)
    #         config_obj = dict(config_files=CONFIG_2_FILE_LIST)
    #         _create_temp_yaml(TEMP_PATH_FOR_YAML, CONFIG_3_NAME, config_obj)
    #
    #         with self.assertRaises(ParserError) as cm:
    #             get_config_dict(config_obj)
    #         self.assertEqual('YAML in \'config_2.json\' is invalid: '
    #                          'while parsing a block mapping\n'
    #                          'expected <block end>, but found \':\'\n'
    #                          '  in "<file>", line 2, column 1',
    #                          f'{cm.exception}')
    #     finally:
    #         if os.path.exists(TEMP_PATH_FOR_YAML):
    #             shutil.rmtree(TEMP_PATH_FOR_YAML)
    #             print('Successfully removed folder')


# test/core/gen/test_gen.py

import os
import unittest
from typing import Any, Dict, Optional, Sequence, Tuple

import numpy as np
import xarray as xr

from test.core.gen.helpers import get_inputdata_path
from xcube.core.dsio import rimraf
from xcube.core.gen.config import get_config_dict
from xcube.core.gen.gen import gen_cube


def clean_up():
    files = ['l2c-single.nc', 'l2c-single.zarr', 'l2c.nc', 'l2c.zarr', 'l2c_presorted.zarr', 'l2c_1x80x60.zarr',
             'l2c_1x80x80.zarr', 'l2c_packed.zarr', 'l2c_packed_1x80x80.zarr']
    for file in files:
        rimraf(file)
        rimraf(file + '.temp.nc')  # May remain from Netcdf4DatasetIO.append()
    rimraf(get_inputdata_path("input.txt"))


class DefaultProcessTest(unittest.TestCase):

    def setUp(self):
        clean_up()

    def tearDown(self):
        clean_up()

    def test_process_inputs_single_nc(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c-single.nc')
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c-single.nc...\n' in output)
        with xr.open_dataset('l2c-single.nc') as dataset:
            self.assert_cube_ok(dataset,
                                expected_time_dim=1,
                                expected_extra_attrs=dict(date_modified=None,
                                                          time_coverage_start='2016-12-31T12:00:00.000000000',
                                                          time_coverage_end='2017-01-01T12:00:00.000000000'))

    def test_process_inputs_single_nc_processed_vars(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c-single.nc',
            processed_variables=(
                ('analysed_sst', dict(valid_pixel_expression=None)),
                ('analysis_error', dict(valid_pixel_expression=None)),
                ('sea_ice_fraction', dict(valid_pixel_expression=None)),
                ('water_mask', dict(expression='(mask.sea or mask.lake) and not mask.ice', load=True)),
                ('ice_mask', dict(expression='mask.sea and mask.ice')),
                ('analysed_sst', dict(valid_pixel_expression='water_mask')),
                ('analysis_error', dict(valid_pixel_expression='water_mask')),
                ('sea_ice_fraction', dict(valid_pixel_expression='ice_mask')),
            ),
            output_variables=(
                ('analysed_sst', dict(name='SST')),
                ('analysis_error', dict(name='SST_uncertainty')),
                ('sea_ice_fraction', None),
            ),
        )
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c-single.nc...\n' in output)
        with xr.open_dataset('l2c-single.nc') as dataset:
            self.assert_cube_ok(dataset,
                                expected_time_dim=1,
                                expected_output_vars=('SST', 'SST_uncertainty', 'sea_ice_fraction'),
                                expected_extra_attrs=dict(date_modified=None,
                                                          time_coverage_start='2016-12-31T12:00:00.000000000',
                                                          time_coverage_end='2017-01-01T12:00:00.000000000'))

    def test_process_inputs_append_multiple_nc(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.nc',
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c.nc...\n' in output)
        self.assertTrue('\nstep 9 of 9: appending input slice to l2c.nc...\n' in output)
        with xr.open_dataset('l2c.nc') as dataset:
            self.assert_cube_ok(dataset,
                                expected_time_dim=3,
                                expected_extra_attrs=dict(date_modified=None,
                                                          time_coverage_start='2016-12-31T12:00:00.000000000',
                                                          time_coverage_end='2017-01-03T12:00:00.000000000'))

    def test_process_inputs_single_zarr(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c-single.zarr')
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c-single.zarr...\n' in output)
        self.assert_cube_ok(xr.open_zarr('l2c-single.zarr'),
                            expected_time_dim=1,
                            expected_extra_attrs=dict(date_modified=None,
                                                      time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-01T12:00:00.000000000'))

    def test_process_inputs_append_multiple_zarr(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.zarr',
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: appending input slice to l2c.zarr...\n' in output)
        self.assert_cube_ok(xr.open_zarr('l2c.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(date_modified=None,
                                                      time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))

    def test_process_inputs_insert_multiple_zarr(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c_presorted.zarr')
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c_presorted.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: appending input slice to l2c_presorted.zarr...\n' in output)
        self.assertFalse('\nstep 9 of 9: inserting input slice before index 0 in l2c_presorted.zarr...\n' in output)
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.zarr',
            no_sort_mode=True)
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: appending input slice to l2c.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: inserting input slice before index 0 in l2c.zarr...\n' in output)
        self.assert_cube_ok(xr.open_zarr('l2c.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(date_modified=None,
                                                      time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        ds_presorted = xr.open_zarr('l2c_presorted.zarr')
        ds_insert = xr.open_zarr('l2c.zarr')
        np.testing.assert_allclose(ds_presorted.analysed_sst.values, ds_insert.analysed_sst.values)

    def test_process_inputs_replace_multiple_zarr(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.zarr',
            no_sort_mode=True)
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: appending input slice to l2c.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: replacing input slice at index 1 in l2c.zarr...\n' in output)
        self.assert_cube_ok(xr.open_zarr('l2c.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(date_modified=None,
                                                      time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        self.assertTrue(os.path.exists(os.path.join('l2c.zarr', '.zmetadata')))

    def test_input_txt(self):
        f = open((os.path.join(os.path.dirname(__file__), 'inputdata', "input.txt")), "w+")
        for i in range(1, 4):
            file_name = f"2017010{i}-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc"
            file = get_inputdata_path(file_name)
            f.write("%s\n" % file)
        f.close()
        status, output = gen_cube_wrapper([get_inputdata_path('input.txt')], 'l2c.zarr', no_sort_mode=False)
        self.assertEqual(True, status)
        self.assert_cube_ok(xr.open_zarr('l2c.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        self.assertTrue(os.path.exists(os.path.join('l2c.zarr', '.zmetadata')))

    def test_process_chunked_zarr(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c.zarr',
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assert_cube_ok(xr.open_zarr('l2c.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c_1x80x60.zarr',
            output_writer_params={'chunksizes': {'lon': 80, 'lat': 60}},
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assert_cube_ok(xr.open_zarr('l2c_1x80x60.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        ds_unchunked = xr.open_zarr('l2c.zarr')
        ds_chunked = xr.open_zarr('l2c_1x80x60.zarr')
        np.testing.assert_allclose(ds_unchunked.analysed_sst.values, ds_chunked.analysed_sst.values)
        self.assertEqual(((1, 1, 1), (60, 60, 60), (80, 80, 80, 80)), ds_chunked.analysed_sst.chunks)

        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c_1x80x80.zarr',
            output_writer_params={'chunksizes': {'lon': 80, 'lat': 80}},
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assert_cube_ok(xr.open_zarr('l2c_1x80x80.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        ds_unchunked = xr.open_zarr('l2c.zarr')
        ds_chunked = xr.open_zarr('l2c_1x80x80.zarr')
        np.testing.assert_allclose(ds_unchunked.analysed_sst.values, ds_chunked.analysed_sst.values)
        self.assertEqual(((1, 1, 1), (80, 80, 20), (80, 80, 80, 80)), ds_chunked.analysed_sst.chunks)

    def test_process_packed_zarr(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c.zarr',
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assert_cube_ok(xr.open_zarr('l2c.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c_packed.zarr',
            output_writer_params={'packing': {'analysed_sst': {'scale_factor': 0.07324442274239326,
                                                               'add_offset': -300.0,
                                                               'dtype': 'uint16',
                                                               '_FillValue': 65535}}},
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assert_cube_ok(xr.open_zarr('l2c_packed.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        ds_unpacked = xr.open_zarr('l2c.zarr')
        ds_packed = xr.open_zarr('l2c_packed.zarr')
        np.testing.assert_almost_equal(ds_unpacked.analysed_sst.values, ds_packed.analysed_sst.values, decimal=1)
        self.assertEqual(0.07324442274239326, ds_packed.analysed_sst.encoding['scale_factor'])
        self.assertEqual(-300.0, ds_packed.analysed_sst.encoding['add_offset'])
        self.assertEqual('uint16', ds_packed.analysed_sst.encoding['dtype'])
        self.assertEqual(65535, ds_packed.analysed_sst.encoding['_FillValue'])
        self.assertEqual((1, 180, 320), ds_packed.analysed_sst.encoding['chunks'])

        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c_packed_1x80x80.zarr',
            output_writer_params={'chunksizes': {'lon': 80, 'lat': 80},
                                  'packing': {'analysed_sst': {'scale_factor': 0.07324442274239326,
                                                               'add_offset': -300.0,
                                                               'dtype': 'uint16',
                                                               '_FillValue': 65535}}},
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assert_cube_ok(xr.open_zarr('l2c_packed_1x80x80.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        ds_unpacked = xr.open_zarr('l2c.zarr')
        ds_packed = xr.open_zarr('l2c_packed_1x80x80.zarr')
        np.testing.assert_almost_equal(ds_unpacked.analysed_sst.values, ds_packed.analysed_sst.values, decimal=1)
        self.assertEqual(((1, 1, 1), (80, 80, 20), (80, 80, 80, 80)), ds_packed.analysed_sst.chunks)
        self.assertEqual(0.07324442274239326, ds_packed.analysed_sst.encoding['scale_factor'])
        self.assertEqual(-300.0, ds_packed.analysed_sst.encoding['add_offset'])
        self.assertEqual('uint16', ds_packed.analysed_sst.encoding['dtype'])
        self.assertEqual(65535, ds_packed.analysed_sst.encoding['_FillValue'])
        self.assertEqual((1, 80, 80), ds_packed.analysed_sst.encoding['chunks'])

    def test_insert_packed_zarr(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c_presorted.zarr',
            output_writer_params={'chunksizes': {'lon': 80, 'lat': 80},
                                  'packing': {'analysed_sst': {'scale_factor': 0.07324442274239326,
                                                               'add_offset': -300.0,
                                                               'dtype': 'uint16',
                                                               '_FillValue': 65535}}})
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c_presorted.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: appending input slice to l2c_presorted.zarr...\n' in output)
        self.assertFalse('\nstep 9 of 9: inserting input slice before index 0 in l2c_presorted.zarr...\n' in output)
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.zarr',
            output_writer_params={'chunksizes': {'lon': 80, 'lat': 80},
                                  'packing': {'analysed_sst': {'scale_factor': 0.07324442274239326,
                                                               'add_offset': -300.0,
                                                               'dtype': 'uint16',
                                                               '_FillValue': 65535}}},
            no_sort_mode=True)
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: appending input slice to l2c.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: inserting input slice before index 0 in l2c.zarr...\n' in output)
        self.assert_cube_ok(xr.open_zarr('l2c.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(date_modified=None,
                                                      time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        ds_presorted = xr.open_zarr('l2c_presorted.zarr')
        ds_insert = xr.open_zarr('l2c.zarr')
        np.testing.assert_allclose(ds_presorted.analysed_sst.values, ds_insert.analysed_sst.values)

    def test_replace_packed_zarr(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c_presorted.zarr',
            output_writer_params={'chunksizes': {'lon': 80, 'lat': 80},
                                  'packing': {'analysed_sst': {'scale_factor': 0.07324442274239326,
                                                               'add_offset': -300.0,
                                                               'dtype': 'uint16',
                                                               '_FillValue': 65535}}})
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c_presorted.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: appending input slice to l2c_presorted.zarr...\n' in output)
        self.assertFalse('\nstep 9 of 9: inserting input slice before index 0 in l2c_presorted.zarr...\n' in output)
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'),
             get_inputdata_path('20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')], 'l2c.zarr',
            output_writer_params={'chunksizes': {'lon': 80, 'lat': 80},
                                  'packing': {'analysed_sst': {'scale_factor': 0.07324442274239326,
                                                               'add_offset': -300.0,
                                                               'dtype': 'uint16',
                                                               '_FillValue': 65535}}},
            no_sort_mode=True)
        self.assertEqual(True, status)
        self.assertTrue('\nstep 9 of 9: creating input slice in l2c.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: appending input slice to l2c.zarr...\n' in output)
        self.assertTrue('\nstep 9 of 9: replacing input slice at index 1 in l2c.zarr...\n' in output)
        self.assert_cube_ok(xr.open_zarr('l2c.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(date_modified=None,
                                                      time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        ds_presorted = xr.open_zarr('l2c_presorted.zarr')
        ds_insert = xr.open_zarr('l2c.zarr')
        np.testing.assert_allclose(ds_presorted.analysed_sst.values, ds_insert.analysed_sst.values)

    def test_process_compressed_zarr(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c.zarr',
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assert_cube_ok(xr.open_zarr('l2c.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        status, output = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c_packed.zarr',
            output_writer_params={'compressor': {'cname': 'zstd', 'clevel': 1, 'shuffle': 2}},
            no_sort_mode=False)
        self.assertEqual(True, status)
        self.assert_cube_ok(xr.open_zarr('l2c_packed.zarr'),
                            expected_time_dim=3,
                            expected_extra_attrs=dict(time_coverage_start='2016-12-31T12:00:00.000000000',
                                                      time_coverage_end='2017-01-03T12:00:00.000000000'))
        ds_default_compressor = xr.open_zarr('l2c.zarr')
        ds_compressed = xr.open_zarr('l2c_packed.zarr')
        np.testing.assert_allclose(ds_default_compressor.analysed_sst.values, ds_compressed.analysed_sst.values)
        self.assertEqual("Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)",
                         str(ds_default_compressor.analysed_sst.encoding['compressor']))
        self.assertEqual("Blosc(cname='zstd', clevel=1, shuffle=BITSHUFFLE, blocksize=0)",
                         str(ds_compressed.analysed_sst.encoding['compressor']))

    def assert_cube_ok(self, cube: xr.Dataset,
                       expected_time_dim: int,
                       expected_extra_attrs: Dict[str, Any],
                       expected_output_vars: Sequence[str] = ('analysed_sst',)):
        self.assertEqual({'lat': 180, 'lon': 320, 'bnds': 2, 'time': expected_time_dim}, cube.dims)
        self.assertEqual({'lon', 'lat', 'time', 'lon_bnds', 'lat_bnds', 'time_bnds'}, set(cube.coords))
        self.assertEqual(set(expected_output_vars), set(cube.data_vars))
        expected_attrs = dict(title='Test Cube',
                              project='xcube',
                              date_modified=None,
                              geospatial_lon_min=-4.0,
                              geospatial_lon_max=12.0,
                              geospatial_lon_resolution=0.05,
                              geospatial_lon_units='degrees_east',
                              geospatial_lat_min=47.0,
                              geospatial_lat_max=56.0,
                              geospatial_lat_resolution=0.05,
                              geospatial_lat_units='degrees_north')
        expected_attrs.update(expected_extra_attrs)
        for k, v in expected_attrs.items():
            self.assertIn(k, cube.attrs)
            if v is not None:
                self.assertEqual(v, cube.attrs[k], msg=f'key {k!r}')

    def test_handle_360_lon(self):
        status, output = gen_cube_wrapper(
            [get_inputdata_path('20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc')],
            'l2c-single.zarr', no_sort_mode=False)
        self.assertEqual(True, status)
        ds = xr.open_zarr('l2c-single.zarr')
        self.assertIn('lon', ds.coords)
        self.assertFalse(np.any(ds.coords['lon'] > 180.))

    def test_illegal_proc(self):
        with self.assertRaises(ValueError) as e:
            gen_cube_wrapper(
                [get_inputdata_path('20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc')],
                'l2c-single.zarr', no_sort_mode=False, input_processor_name="")
        self.assertEqual('input_processor_name must not be empty', f'{e.exception}')

        with self.assertRaises(ValueError) as e:
            gen_cube_wrapper(
                [get_inputdata_path('20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc')],
                'l2c-single.zarr', no_sort_mode=False, input_processor_name='chris-proba')
        self.assertEqual("Unknown input_processor_name 'chris-proba'", f'{e.exception}')


# noinspection PyShadowingBuiltins
def gen_cube_wrapper(input_paths,
                     output_path,
                     output_writer_params=None,
                     no_sort_mode=False,
                     input_processor_name=None,
                     processed_variables=None,
                     output_variables=(('analysed_sst', None),),
                     ) -> Tuple[bool, Optional[str]]:
    output = None

    def output_monitor(msg):
        nonlocal output
        if output is None:
            output = msg + '\n'
        else:
            output += msg + '\n'

    config = get_config_dict(
        input_paths=input_paths,
        input_processor_name=input_processor_name,
        output_path=output_path,
        output_size='320,180',
        output_region='-4,47,12,56',
        output_resampling='Nearest',
        no_sort_mode=no_sort_mode,
    )
    if processed_variables is not None:
        config.update(processed_variables=processed_variables)
    if output_variables is not None:
        config.update(output_variables=output_variables)
    if output_writer_params is not None:
        config.update(output_writer_params=output_writer_params)

    output_metadata = dict(
        title='Test Cube',
        project='xcube',
    )

    return gen_cube(dry_run=False, monitor=output_monitor, output_metadata=output_metadata, **config), output


# test/core/gen/test_iproc.py

import unittest

import numpy as np
import pandas as pd
import xarray as xr

from xcube.core.gen.iproc import DefaultInputProcessor, ReprojectionInfo
from xcube.core.timecoord import to_time_in_days_since_1970


class ReprojectionInfoTest(unittest.TestCase):
    def test_success(self):
        # noinspection PyTypeChecker
        ri = ReprojectionInfo(xy_names=['x', 'y'])
        self.assertEqual(('x', 'y'), ri.xy_names)
        self.assertEqual(None, ri.xy_tp_names)
        self.assertEqual(None, ri.xy_crs)
        self.assertEqual(None, ri.xy_gcp_step)
        self.assertEqual(None, ri.xy_tp_gcp_step)

        # noinspection PyTypeChecker
        ri = ReprojectionInfo(xy_names=['x', 'y'], xy_gcp_step=[2, 1])
        self.assertEqual(('x', 'y'), ri.xy_names)
        self.assertEqual(None, ri.xy_tp_names)
        self.assertEqual(None, ri.xy_crs)
        self.assertEqual((2, 1), ri.xy_gcp_step)
        self.assertEqual(None, ri.xy_tp_gcp_step)

        # noinspection PyTypeChecker
        ri = ReprojectionInfo(xy_gcp_step=4, xy_tp_gcp_step=2)
        self.assertEqual(None, ri.xy_names)
        self.assertEqual(None, ri.xy_tp_names)
        self.assertEqual(None, ri.xy_crs)
        self.assertEqual((4, 4), ri.xy_gcp_step)
        self.assertEqual((2, 2), ri.xy_tp_gcp_step)

    def test_fail(self):
        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            ReprojectionInfo(xy_names=['x', ''])
        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            ReprojectionInfo(xy_names=[0, 'y'])
        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            ReprojectionInfo(xy_gcp_step=[7, 'y'])
        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            ReprojectionInfo(xy_gcp_step=[7, 0])


class DefaultInputProcessorTest(unittest.TestCase):

    def setUp(self):
        self.processor = DefaultInputProcessor()

    def test_props(self):
        self.assertEqual('default', self.processor.name)
        self.assertEqual('Single-scene NetCDF/CF inputs in xcube standard format', self.processor.description)
        self.assertEqual('netcdf4', self.processor.input_reader)

        processor = DefaultInputProcessor(input_reader="zarr")
        self.assertEqual('zarr', processor.input_reader)

    def test_reprojection_info(self):
        # noinspection PyNoneFunctionAssignment
        reprojection_info = self.processor.get_reprojection_info(create_default_dataset())
        self.assertIsNotNone(reprojection_info)

    def test_get_time_range(self):
        ds = create_default_dataset(time_mode="time")
        t1, t2 = self.processor.get_time_range(ds)
        self.assertEqual(to_time_in_days_since_1970("20100301T120000Z"), t1)
        self.assertEqual(to_time_in_days_since_1970("20100301T120000Z"), t2)
        ds = create_default_dataset(time_mode="time_bnds")
        t1, t2 = self.processor.get_time_range(ds)
        self.assertEqual(to_time_in_days_since_1970("20100301T000000Z"), t1)
        self.assertEqual(to_time_in_days_since_1970("20100301T235959Z"), t2)
        ds = create_default_dataset(time_mode="time_coverage")
        t1, t2 = self.processor.get_time_range(ds)
        self.assertEqual(to_time_in_days_since_1970("20100301T000000Z"), t1)
        self.assertEqual(to_time_in_days_since_1970("20100301T235959Z"), t2)
        ds = create_default_dataset(time_mode="start_stop_time")
        t1, t2 = self.processor.get_time_range(ds)
        self.assertEqual(to_time_in_days_since_1970("20100301T000000Z"), t1)
        self.assertEqual(to_time_in_days_since_1970("20100301T235959Z"), t2)
        ds = create_default_dataset(time_mode="start_stop_date")
        t1, t2 = self.processor.get_time_range(ds)
        self.assertEqual(to_time_in_days_since_1970("20100301T000000Z"), t1)
        self.assertEqual(to_time_in_days_since_1970("20100301T235959Z"), t2)
        ds = create_default_dataset(time_mode="no_time")
        with self.assertRaises(ValueError) as cm:
            self.processor.get_time_range(ds)
        self.assertEqual("invalid input: missing time coverage information in dataset", f"{cm.exception}")

    def test_pre_process(self):
        ds1 = create_default_dataset(time_mode="time")
        ds2 = self.processor.pre_process(ds1)
        self.assertIsNot(ds1, ds2)
        ds1 = create_default_dataset(time_mode="time_bnds")
        ds2 = self.processor.pre_process(ds1)
        self.assertIsNot(ds1, ds2)
        ds1 = create_default_dataset(time_mode="time_coverage")
        ds2 = self.processor.pre_process(ds1)
        self.assertIs(ds1, ds2)
        ds1 = create_default_dataset(time_mode="start_stop_time")
        ds2 = self.processor.pre_process(ds1)
        self.assertIs(ds1, ds2)

    def test_post_process(self):
        ds1 = create_default_dataset()
        ds2 = self.processor.post_process(ds1)
        self.assertIs(ds1, ds2)


def create_default_dataset(time_mode: str = "time_bnds"):
    w = 7200
    h = 3600
    res = 180. / h
    lon = np.linspace(-180 + 0.5 * res, 180 - 0.5 * res, w)
    lat = np.linspace(-90 + 0.5 * res, 90 - 0.5 * res, h)
    time = np.array([pd.to_datetime("20100301T120000")], dtype="datetime64[ns]")
    time_bnds = np.array([[pd.to_datetime("20100301T000000"), pd.to_datetime("20100301T235959")]],
                         dtype="datetime64[ns]")

    coords = dict(
        lon=(("lon",), lon, dict(long_name="longitude", units="degrees_east")),
        lat=(("lat",), lat, dict(long_name="latitude", units="degrees_north")),
    )

    if time_mode == "time":
        coords.update(dict(
            time=(("time",), time,
                  dict(long_name="time", units="nanoseconds since 1970-01-01"))
        ))
        var_dims = ("time", "lat", "lon")
        var_shape = (1, h, w)
    elif time_mode == "time_bnds":
        coords.update(dict(
            time=(
                ("time",), time,
                dict(long_name="time", units="nanoseconds since 1970-01-01")),
            time_bnds=(
                ("time", "bnds"), time_bnds,
                dict(long_name="time bounds", units="nanoseconds since 1970-01-01")),
        ))
        var_dims = ("time", "lat", "lon")
        var_shape = (1, h, w)
    else:
        var_dims = ("lat", "lon")
        var_shape = (h, w)

    analysed_sst = np.zeros(shape=var_shape, dtype=np.float32)
    analysis_error = np.zeros(shape=var_shape, dtype=np.float32)
    mask = np.zeros(shape=var_shape, dtype=np.int32)
    data_vars = dict(
        analysed_sst=(var_dims, analysed_sst),
        analysis_error=(var_dims, analysis_error),
        mask=(var_dims, mask),
    )

    attrs = dict([
        ('title', 'ESA SST CCI OSTIA L4 product'),
        ('institution', 'ESACCI'),
        ('publisher_name', 'ESACCI'),
        ('processing_level', 'L4'),
        ('Metadata_Conventions', 'Unidata Dataset Discovery v1.0'),
        ('Conventions', 'CF-1.5, Unidata Observation Dataset v1.0'),
        ('geospatial_lat_max', 90.0),
        ('geospatial_lat_min', -90.0),
        ('geospatial_lon_max', 180.0),
        ('geospatial_lon_min', -180.0),
    ])
    if time_mode == "time_coverage":
        attrs.update(dict([
            ('time_coverage_start', '20100301T000000Z'),
            ('time_coverage_end', '20100301T235959Z'),
            ('time_coverage_duration', 'P1D'),
            ('time_coverage_resolution', 'P1D'),
        ]))
    elif time_mode == "start_stop_time":
        attrs.update(dict([
            ('start_time', '20100301T000000Z'),
            ('stop_time', '20100301T235959Z'),
        ]))

    elif time_mode == "start_stop_date":
        attrs.update(dict([
            ('start_date', '01-MAR-2010 00:00:00.000000'),
            ('stop_date', '01-MAR-2010 23:59:59.000000'),
        ]))

    return xr.Dataset(coords=coords, data_vars=data_vars, attrs=attrs)


# test/core/gen2/test_config.py

import json
import unittest

from xcube.core.gen2.config import CallbackConfig
from xcube.core.gen2.config import CubeConfig
from xcube.core.gen2.config import InputConfig
from xcube.core.gen2.config import OutputConfig


class InputConfigTest(unittest.TestCase):

    def test_from_dict(self):
        json_instance = dict(store_id='sentinelhub', data_id='S2L2A')
        input_config = InputConfig.from_dict(json_instance)
        self.assertIsInstance(input_config, InputConfig)
        self.assertEqual('sentinelhub', input_config.store_id)
        self.assertEqual('S2L2A', input_config.data_id)
        self.assertEqual(None, input_config.store_params)
        self.assertEqual(None, input_config.open_params)

    def test_to_dict(self):
        expected_dict = dict(store_id='sentinelhub', data_id='S2L2A')
        input_config = InputConfig(**expected_dict)
        actual_dict = input_config.to_dict()
        self.assertEqual(expected_dict, actual_dict)
        # smoke test JSON serialisation
        json.dumps(actual_dict, indent=2)


class OutputConfigTest(unittest.TestCase):

    def test_from_dict(self):
        json_instance = dict(store_id='s3', data_id='CHL.zarr')
        output_config = OutputConfig.get_schema().from_instance(json_instance)
        self.assertIsInstance(output_config, OutputConfig)
        self.assertEqual('s3', output_config.store_id)
        self.assertEqual('CHL.zarr', output_config.data_id)
        self.assertEqual(None, output_config.store_params)
        self.assertEqual(None, output_config.write_params)

    def test_to_dict(self):
        expected_dict = dict(store_id='s3', replace=False, data_id='CHL.zarr')
        output_config = OutputConfig(**expected_dict)
        actual_dict = output_config.to_dict()
        self.assertEqual(expected_dict, actual_dict)
        # smoke test JSON serialisation
        json.dumps(actual_dict, indent=2)


class CubeConfigTest(unittest.TestCase):

    def test_from_dict(self):
        json_instance = dict(variable_names=['B03', 'B04'],
                             crs='WGS84',
                             bbox=[12.2, 52.1, 13.9, 54.8],
                             spatial_res=0.05,
                             time_range=['2018-01-01', None],
                             time_period='4D',
                             metadata=dict(title='S2L2A subset'),
                             variable_metadata=dict(
                                 B03=dict(long_name='Band 3'),
                                 B04=dict(long_name='Band 4'),
                             ))
        cube_config = CubeConfig.from_dict(json_instance)
        self.assertIsInstance(cube_config, CubeConfig)
        self.assertEqual(('B03', 'B04'), cube_config.variable_names)
        self.assertEqual('WGS84', cube_config.crs)
        self.assertEqual((12.2, 52.1, 13.9, 54.8), cube_config.bbox)
        self.assertEqual(0.05, cube_config.spatial_res)
        self.assertEqual(('2018-01-01', None), cube_config.time_range)
        self.assertEqual('4D', cube_config.time_period)
        self.assertEqual(dict(title='S2L2A subset'),
                         cube_config.metadata)
        self.assertEqual(
            dict(
                B03=dict(long_name='Band 3'),
                B04=dict(long_name='Band 4'),
            ),
            cube_config.variable_metadata
        )

    def test_to_dict(self):
        expected_dict = dict(variable_names=['B03', 'B04'],
                             crs='WGS84',
                             bbox=[12.2, 52.1, 13.9, 54.8],
                             spatial_res=0.05,
                             time_range=['2018-01-01', None],
                             time_period='4D',
                             metadata=dict(title='S2L2A subset'),
                             variable_metadata=dict(
                                 B03=dict(long_name='Band 3'),
                                 B04=dict(long_name='Band 4'),
                             ))
        cube_config = CubeConfig.get_schema().from_instance(expected_dict)
        actual_dict = cube_config.to_dict()
        self.assertEqual(expected_dict, actual_dict)

        # smoke test JSON serialisation
        json.dumps(actual_dict, indent=2)

    def test_drop_props(self):
        cube_config = CubeConfig(variable_names=['B03', 'B04'],
                                 crs='WGS84',
                                 bbox=(12.2, 52.1, 13.9, 54.8),
                                 spatial_res=0.05,
                                 time_range=('2018-01-01', None),
                                 time_period='4D')

        self.assertIs(cube_config, cube_config.drop_props([]))

        lesser_cube_config = cube_config.drop_props(['crs',
                                                     'spatial_res',
                                                     'bbox'])
        self.assertEqual(None, lesser_cube_config.crs)
        self.assertEqual(None, lesser_cube_config.bbox)
        self.assertEqual(None, lesser_cube_config.spatial_res)
        self.assertEqual(('B03', 'B04'), lesser_cube_config.variable_names)
        self.assertEqual(('2018-01-01', None), lesser_cube_config.time_range)
        self.assertEqual('4D', lesser_cube_config.time_period)

        lesser_cube_config = cube_config.drop_props('variable_names')
        self.assertEqual(None, lesser_cube_config.variable_names)

        with self.assertRaises(ValueError) as cm:
            cube_config.drop_props('variable_shames')
        self.assertEqual("variable_shames is not a property of"
                         " <class 'xcube.core.gen2.config.CubeConfig'>",
                         f'{cm.exception}')


class CallbackConfigTest(unittest.TestCase):

    def test_to_dict(self):
        with self.assertRaises(ValueError) as e:
            CallbackConfig()
        self.assertEqual('Both, api_uri and access_token must be given',
                         str(e.exception))

        expected_dict = {
            "api_uri": 'https://bla.com',
            "access_token": 'dfasovjdasoövjidfs'
        }
        callback = CallbackConfig(**expected_dict)
        actual_dict = callback.to_dict()

        self.assertDictEqual(expected_dict, actual_dict)

        # smoke test JSON serialisation
        json.dumps(actual_dict, indent=2)


# test/core/gen2/test_progress.py

import unittest
from unittest.mock import patch

import requests_mock

from xcube.core.gen2.request import CubeGeneratorRequest
from xcube.core.gen2.progress import ApiProgressCallbackObserver
from xcube.core.gen2.progress import TerminalProgressCallbackObserver
from xcube.core.gen2.progress import _ThreadedProgressObserver
from xcube.util.progress import ProgressState


class TestThreadedProgressObservers(unittest.TestCase):
    REQUEST = dict(input_config=dict(store_id='sentinelhub',
                                     data_id='S2L2A',
                                     ),
                   cube_config=dict(variable_names=['B01', 'B02', 'B03'],
                                    crs='WGS84',
                                    bbox=[12.2, 52.1, 13.9, 54.8],
                                    spatial_res=0.05,
                                    time_range=['2018-01-01', None],
                                    time_period='4D'),
                   output_config=dict(store_id='memory',
                                      data_id='CHL'),
                   callback_config=dict(api_uri='https://xcube-gen.test/api/v1/jobs/tomtom/iamajob/callback',
                                        access_token='dfsvdfsv'))

    def setUp(self) -> None:
        self._request = CubeGeneratorRequest.from_dict(self.REQUEST)
        self._callback_config = self._request.callback_config

    @requests_mock.Mocker()
    def test_api_delegate(self, m):
        m.put('https://xcube-gen.test/api/v1/jobs/tomtom/iamajob/callback', json={})
        progress_state = ProgressState(label='test', total_work=0., super_work=10.)

        observer = ApiProgressCallbackObserver(self._callback_config)
        observer.on_begin(state_stack=[progress_state])

        with self.assertRaises(ValueError) as e:
            self._callback_config.api_uri = None
            observer = ApiProgressCallbackObserver(self._callback_config)
            observer.on_begin(state_stack=[progress_state])

        self.assertEqual('Both, api_uri and access_token must be given.', str(e.exception))

        with self.assertRaises(ValueError) as e:
            self._callback_config.access_token = None
            observer = ApiProgressCallbackObserver(self._callback_config)
            observer.on_begin(state_stack=[progress_state])

        self.assertEqual('Both, api_uri and access_token must be given.', str(e.exception))

    @requests_mock.Mocker()
    def test_callback(self, m):
        expected_callback = {
            "sender": "on_begin",
            "state": {
                "label": "Test",
                "total_work": 100,
                "error": False,
                "progress": 0.0,
                "elapsed": 3.,
            }
        }

        m.put('https://xcube-gen.test/api/v1/jobs/tomtom/iamajob/callback', json=expected_callback)

        state_stack = ProgressState('Test', 100, 100)

        observer = ApiProgressCallbackObserver(self._callback_config)

        res = observer.callback("on_begin", 3., [state_stack])
        self.assertDictEqual(expected_callback, res.request.json())

        with self.assertRaises(ValueError) as e:
            observer.callback("on_begin", 3., [])

        self.assertEqual("ProgressStates must be given", str(e.exception))

        observer = TerminalProgressCallbackObserver()
        res = observer.callback("on_begin", 3., [state_stack], False)
        self.assertIn('Test', res)
        self.assertIn('0% Completed', res)

    def test_threaded_progress_on_begin(self):
        _mock_patch = patch('xcube.core.gen2.progress._ThreadedProgressObserver._start_timer')
        _mock = _mock_patch.start()

        observer = TerminalProgressCallbackObserver()

        state_stack = ProgressState('Test', 100, 100)

        observer.on_begin([state_stack])
        self.assertTrue(_mock.called)
        _mock.stop()

        _mock = _mock_patch.start()
        state_stack1 = ProgressState('Test', 100, 100)
        state_stack2 = ProgressState('Test', 100, 100)

        observer.on_begin([state_stack1, state_stack2])
        self.assertFalse(_mock.called)

    def test_threaded_progress_on_end(self):
        _mock_patch = patch('xcube.core.gen2.progress._ThreadedProgressObserver._stop_timer')
        _mock = _mock_patch.start()

        observer = TerminalProgressCallbackObserver()
        observer._running = True

        state_stack = ProgressState('Test', 100, 100)

        observer.on_end([state_stack])
        self.assertTrue(_mock.called)
        _mock.stop()

        _mock = _mock_patch.start()
        state_stack1 = ProgressState('Test', 100, 100)
        state_stack2 = ProgressState('Test', 100, 100)

        observer.on_end([state_stack1, state_stack2])
        self.assertFalse(_mock.called)

    # def test_running_progress(self):
    #     """
    #     Uncomment the lines below if you want to run and test the termial progress bar output.
    #     """

    # from time import sleep
    # from xcube.util.progress import observe_progress
    #
    # TerminalProgressCallbackObserver().activate()
    #
    # with observe_progress('Generating cube', 100) as cm:
    #     dt = 1
    #     for i in range(1, 80):
    #         cm.will_work(1)
    #         sleep(dt)
    #         cm.worked(1)
    #
    #     cm.will_work(20)
    #     sleep(dt)
    #     cm.worked(20)


class TestThreadedProgressObserver(unittest.TestCase):
    def test_threaded(self):
        with self.assertRaises(ValueError) as e:
            _ThreadedProgressObserver(minimum=-1, dt=0)

        self.assertEqual("The timer's minimum must be >=0", str(e.exception))

        with self.assertRaises(ValueError) as e:
            _ThreadedProgressObserver(minimum=0, dt=-1)

        self.assertEqual("The timer's time step must be >=0", str(e.exception))

        observer = _ThreadedProgressObserver(minimum=0, dt=0)

        with self.assertRaises(ValueError) as e:
            observer.on_begin(state_stack=[])

        self.assertEqual('state_stack must be given', str(e.exception))

        with self.assertRaises(ValueError) as e:
            observer.on_update(state_stack=[])

        self.assertEqual('state_stack must be given', str(e.exception))

        with self.assertRaises(ValueError) as e:
            observer.on_end(state_stack=[])

        self.assertEqual('state_stack must be given', str(e.exception))


# test/core/gen2/test_request.py

import json
import os
import unittest

from xcube.core.gen2.config import CubeConfig
from xcube.core.gen2.config import InputConfig
from xcube.core.gen2.config import OutputConfig
from xcube.core.gen2.request import CubeGeneratorRequest


class CubeGeneratorRequestTest(unittest.TestCase):
    def test_normalize(self):
        request_dict = dict(
            input_config=dict(data_id='S2L2A', store_id='memory'),
            cube_config=dict(variable_names=['B01', 'B02'],
                             crs='WGS84',
                             bbox=[12.2, 52.1, 13.9, 54.8],
                             spatial_res=0.05,
                             time_range=['2018-01-01', None],
                             time_period='4D'),
            output_config=dict(store_id='memory',
                               replace=False,
                               data_id='CHL')
        )
        file_path = '_test-request.json'
        with open(file_path, 'w') as fp:
            json.dump(request_dict, fp)
        try:
            request_1 = CubeGeneratorRequest.normalize(file_path)
            self.assertIsInstance(request_1, CubeGeneratorRequest)
            request_2 = CubeGeneratorRequest.normalize(request_1)
            self.assertIs(request_2, request_1)
            request_3 = CubeGeneratorRequest.normalize(request_2.to_dict())
            self.assertIsInstance(request_3, CubeGeneratorRequest)
            self.assertEqual(request_dict, request_3.to_dict())
        finally:
            os.remove(file_path)

        with self.assertRaises(TypeError):
            # noinspection PyTypeChecker
            CubeGeneratorRequest.normalize(42)

    def test_from_dict(self):
        request_dict = dict(
            input_configs=[dict(store_id='memory',
                                data_id='S2L2A')],
            cube_config=dict(variable_names=['B01', 'B02'],
                             crs='WGS84',
                             bbox=[12.2, 52.1, 13.9, 54.8],
                             spatial_res=0.05,
                             time_range=['2018-01-01', None],
                             time_period='4D'),
            output_config=dict(store_id='memory',
                               data_id='CHL')
        )
        gen_config = CubeGeneratorRequest.from_dict(request_dict)
        self.assertIsInstance(gen_config, CubeGeneratorRequest)
        self.assertEqual(1, len(gen_config.input_configs))
        self.assertIsInstance(gen_config.input_configs[0], InputConfig)
        self.assertEqual('memory', gen_config.input_configs[0].store_id)
        self.assertEqual('S2L2A', gen_config.input_configs[0].data_id)
        self.assertIsInstance(gen_config.output_config, OutputConfig)
        self.assertEqual('memory', gen_config.output_config.store_id)
        self.assertEqual('CHL', gen_config.output_config.data_id)
        self.assertIsInstance(gen_config.cube_config, CubeConfig)
        self.assertEqual(('B01', 'B02'), gen_config.cube_config.variable_names)
        self.assertEqual('WGS84', gen_config.cube_config.crs)
        self.assertEqual((12.2, 52.1, 13.9, 54.8), gen_config.cube_config.bbox)
        self.assertEqual(0.05, gen_config.cube_config.spatial_res)
        self.assertEqual(('2018-01-01', None), gen_config.cube_config.time_range)
        self.assertEqual('4D', gen_config.cube_config.time_period)

    def test_to_dict(self):
        expected_dict = dict(
            input_config=dict(store_id='memory',
                              data_id='S2L2A'),
            cube_config=dict(variable_names=['B01', 'B02'],
                             crs='WGS84',
                             bbox=[12.2, 52.1, 13.9, 54.8],
                             spatial_res=0.05,
                             time_range=['2018-01-01', None],
                             time_period='4D'),
            output_config=dict(store_id='memory',
                               replace=False,
                               data_id='CHL')
        )
        request = CubeGeneratorRequest.from_dict(expected_dict)
        actual_dict = request.to_dict()
        self.assertEqual(expected_dict, actual_dict)
        # JSON-serialisation smoke test
        json.dumps(actual_dict, indent=2)

        # No cube config is ok too
        expected_dict = dict(
            input_config=dict(store_id='memory',
                              data_id='S2L2A'),
            output_config=dict(store_id='memory',
                               replace=False,
                               data_id='CHL')
        )
        request = CubeGeneratorRequest.from_dict(expected_dict)
        actual_dict = request.to_dict()
        self.assertEqual(expected_dict, actual_dict)
        # JSON-serialisation smoke test
        json.dumps(actual_dict, indent=2)


# test/core/gen2/test_response.py

import unittest

from xcube.core.gen2.response import CubeGeneratorResult
from xcube.core.gen2.response import CubeInfo
from xcube.core.gen2.response import CubeInfoResult
from xcube.core.gen2.response import CubeReference
from xcube.core.store import DatasetDescriptor


class CubeGeneratorResultTest(unittest.TestCase):
    def test_serialisation(self):
        result = CubeGeneratorResult(status='ok',
                                     message='Success!',
                                     result=CubeReference(data_id='bibo.zarr'))

        result_dict = result.to_dict()
        self.assertIsInstance(result_dict, dict)
        self.assertEqual(
            {
                'status': 'ok',
                'message': 'Success!',
                'result': {'data_id': 'bibo.zarr'},
            },
            result_dict)

        result2 = CubeGeneratorResult.from_dict(result_dict)
        self.assertIsInstance(result2, CubeGeneratorResult)
        self.assertIsInstance(result2.result, CubeReference)
        self.assertEqual('bibo.zarr', result2.result.data_id)


class CubeInfoResultTest(unittest.TestCase):
    def test_serialisation(self):
        result = CubeInfoResult(
            status='ok',
            message='Success!',
            result=CubeInfo(
                dataset_descriptor=DatasetDescriptor(data_id='bibo.zarr'),
                size_estimation={}
            )
        )

        result_dict = result.to_dict()
        self.assertIsInstance(result_dict, dict)
        self.assertEqual(
            {
                'status': 'ok',
                'message': 'Success!',
                'result': {
                    'dataset_descriptor': {
                        'data_id': 'bibo.zarr',
                        'data_type': 'dataset'
                    },
                    'size_estimation': {}
                },
            },
            result_dict)

        result2 = CubeInfoResult.from_dict(result_dict)
        self.assertIsInstance(result2, CubeInfoResult)


# test/core/gridmapping/test_base.py

import unittest

import numpy as np
import pyproj
import xarray as xr

from test.sampledata import SourceDatasetMixin
from xcube.core.gridmapping import GridMapping
from xcube.core.gridmapping.coords import Coords2DGridMapping
# noinspection PyProtectedMember
from xcube.core.gridmapping.helpers import _to_affine
from xcube.core.gridmapping.regular import RegularGridMapping
from xcube.util.tilegrid import TileGrid

GEO_CRS = pyproj.crs.CRS(4326)
NOT_A_GEO_CRS = pyproj.crs.CRS(5243)


class TestGridMapping(GridMapping):
    def _new_xy_coords(self) -> xr.DataArray:
        return GridMapping.regular(size=self.size,
                                   tile_size=self.tile_size,
                                   is_j_axis_up=self.is_j_axis_up,
                                   xy_res=self.xy_res,
                                   xy_min=(self.xy_bbox[0], self.xy_bbox[1]),
                                   crs=self.crs).xy_coords


# noinspection PyMethodMayBeStatic
class GridMappingTest(SourceDatasetMixin, unittest.TestCase):
    _kwargs = dict(
        size=(720, 360),
        tile_size=(360, 180),
        xy_bbox=(-180.0, -90.0, 180.0, 90.0),
        xy_res=(360 / 720, 360 / 720),
        crs=GEO_CRS,
        xy_var_names=('x', 'y'),
        xy_dim_names=('x', 'y'),
        is_regular=True,
        is_lon_360=False,
        is_j_axis_up=False,
    )

    def kwargs(self, **kwargs):
        orig_kwargs = dict(self._kwargs)
        orig_kwargs.update(**kwargs)
        if 'xy_min' in orig_kwargs:
            # Replace xy_min by xy_bbox.
            # Using xy_min instead of xy_bbox makes it easier for us
            width, height = orig_kwargs['size']
            try:
                x_res, y_res = orig_kwargs['xy_res']
            except TypeError:
                x_res, y_res = 2 * (orig_kwargs['xy_res'],)
            x_min, y_min = orig_kwargs.pop('xy_min')
            x_max, y_max = x_min + x_res * width, y_min + y_res * height
            orig_kwargs['xy_bbox'] = x_min, y_min, x_max, y_max
        return orig_kwargs

    def test_valid(self):
        gm = TestGridMapping(**self.kwargs())
        self.assertEqual((720, 360), gm.size)
        self.assertEqual(720, gm.width)
        self.assertEqual(360, gm.height)
        self.assertEqual(True, gm.is_tiled)
        self.assertEqual((360, 180), gm.tile_size)
        self.assertEqual(360, gm.tile_width)
        self.assertEqual(180, gm.tile_height)
        self.assertEqual((0, 0, 720, 360), gm.ij_bbox)
        self.assertEqual((-180.0, -90.0, 180.0, 90.0), gm.xy_bbox)
        self.assertEqual(-180.0, gm.x_min)
        self.assertEqual(-90.0, gm.y_min)
        self.assertEqual(180.0, gm.x_max)
        self.assertEqual(90.0, gm.y_max)
        self.assertEqual((0.5, 0.5), gm.xy_res)
        self.assertEqual(0.5, gm.x_res)
        self.assertEqual(0.5, gm.y_res)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual('degree', gm.spatial_unit_name)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_lon_360)
        self.assertEqual(False, gm.is_j_axis_up)

        self.assertIsInstance(gm.xy_coords, xr.DataArray)
        np.testing.assert_equal(np.array([[0, 0, 360, 180],
                                          [360, 0, 720, 180],
                                          [0, 180, 360, 360],
                                          [360, 180, 720, 360]]), gm.ij_bboxes)
        np.testing.assert_equal(np.array([[-180., 0., 0., 90.],
                                          [0., 0., 180., 90.],
                                          [-180., -90., 0., 0.],
                                          [0., -90., 180., 0.]]), gm.xy_bboxes)

    def test_invalids(self):
        with self.assertRaises(ValueError) as cm:
            TestGridMapping(**self.kwargs(size=(360, 1)))
        self.assertEqual('invalid size', f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            TestGridMapping(**self.kwargs(size=(360,)))
        self.assertEqual('not enough values to unpack (expected 2, got 1)', f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            TestGridMapping(**self.kwargs(size=None))
        self.assertEqual('size must be an int or a sequence of two ints', f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            TestGridMapping(**self.kwargs(tile_size=0))
        self.assertEqual('invalid tile_size', f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            TestGridMapping(**self.kwargs(xy_res=-0.1))
        self.assertEqual('invalid xy_res', f'{cm.exception}')

    def test_scalars(self):
        gm = TestGridMapping(**self.kwargs(size=360, tile_size=180, xy_res=0.1))
        self.assertEqual((360, 360), gm.size)
        self.assertEqual((180, 180), gm.tile_size)
        self.assertEqual((0.1, 0.1), gm.xy_res)

    def test_not_tiled(self):
        gm = TestGridMapping(**self.kwargs(tile_size=None))
        self.assertEqual((720, 360), gm.tile_size)
        self.assertEqual(False, gm.is_tiled)

    def test_ij_to_xy_transform(self):
        image_geom = TestGridMapping(**self.kwargs(size=(1200, 1200),
                                                   xy_min=(0, 0),
                                                   xy_res=1,
                                                   crs=NOT_A_GEO_CRS))
        i2crs = image_geom.ij_to_xy_transform
        self.assertMatrixPoint((0, 0), i2crs, (0, 1200))
        self.assertMatrixPoint((1024, 0), i2crs, (1024, 1200))
        self.assertMatrixPoint((0, 1024), i2crs, (0, 1200 - 1024))
        self.assertMatrixPoint((1024, 1024), i2crs, (1024, 1200 - 1024))
        self.assertEqual(((1, 0, 0), (0.0, -1, 1200)), i2crs)

        image_geom = TestGridMapping(**self.kwargs(size=(1440, 720),
                                                   xy_min=(-180, -90),
                                                   xy_res=0.25))
        i2crs = image_geom.ij_to_xy_transform
        self.assertMatrixPoint((-180, 90), i2crs, (0, 0))
        self.assertMatrixPoint((0, 0), i2crs, (720, 360))
        self.assertMatrixPoint((180, -90), i2crs, (1440, 720))
        self.assertEqual(((0.25, 0.0, -180.0), (0.0, -0.25, 90.0)), i2crs)

        image_geom = TestGridMapping(**self.kwargs(size=(1440, 720),
                                                   xy_min=(-180, -90),
                                                   xy_res=0.25,
                                                   is_j_axis_up=True))
        i2crs = image_geom.ij_to_xy_transform
        self.assertMatrixPoint((-180, -90), i2crs, (0, 0))
        self.assertMatrixPoint((0, 0), i2crs, (720, 360))
        self.assertMatrixPoint((180, 90), i2crs, (1440, 720))
        self.assertEqual(((0.25, 0.0, -180.0), (0.0, 0.25, -90.0)), i2crs)

    def test_xy_to_ij_transform(self):
        image_geom = TestGridMapping(**self.kwargs(size=(1200, 1200),
                                                   xy_min=(0, 0),
                                                   xy_res=1,
                                                   crs=NOT_A_GEO_CRS))
        crs2i = image_geom.xy_to_ij_transform
        self.assertMatrixPoint((0, 0), crs2i, (0, 1200))
        self.assertMatrixPoint((1024, 0), crs2i, (1024, 1200))
        self.assertMatrixPoint((0, 1024), crs2i, (0, 1200 - 1024))
        self.assertMatrixPoint((1024, 1024), crs2i, (1024, 1200 - 1024))
        self.assertEqual(((1, 0, 0), (0.0, -1, 1200)), crs2i)

        image_geom = TestGridMapping(**self.kwargs(size=(1440, 720),
                                                   xy_res=0.25))
        crs2i = image_geom.xy_to_ij_transform
        self.assertMatrixPoint((0, 720), crs2i, (-180, -90))
        self.assertMatrixPoint((720, 360), crs2i, (0, 0))
        self.assertMatrixPoint((1440, 0), crs2i, (180, 90))
        self.assertEqual(((4.0, 0.0, 720.0), (0.0, -4.0, 360.0)), crs2i)

        image_geom = TestGridMapping(**self.kwargs(size=(1440, 720),
                                                   xy_res=0.25,
                                                   is_j_axis_up=True))
        crs2i = image_geom.xy_to_ij_transform
        self.assertMatrixPoint((0, 0), crs2i, (-180, -90))
        self.assertMatrixPoint((720, 360), crs2i, (0, 0))
        self.assertMatrixPoint((1440, 720), crs2i, (180, 90))
        self.assertEqual(((4.0, 0.0, 720.0), (0.0, 4.0, 360.0)), crs2i)

    def test_ij_transform_to_and_from(self):
        gm1 = TestGridMapping(**self.kwargs(size=(1440, 720),
                                            xy_res=0.25,
                                            is_j_axis_up=True))
        gm2 = TestGridMapping(**self.kwargs(size=(1000, 1000),
                                            xy_min=(10, 50),
                                            xy_res=0.025,
                                            is_j_axis_up=True))
        self.assertEqual(((10.0, 0.0, -7600.0),
                          (0.0, 10.0, -5600.0)),
                         gm1.ij_transform_to(gm2))
        self.assertEqual(((10.0, 0.0, -7600.0),
                          (0.0, 10.0, -5600.0)),
                         gm2.ij_transform_from(gm1))
        self.assertEqual(((0.1, 0.0, 760.0),
                          (0.0, 0.1, 560.0)),
                         gm2.ij_transform_to(gm1))
        self.assertEqual(((0.1, 0.0, 760.0),
                          (0.0, 0.1, 560.0)),
                         gm1.ij_transform_from(gm2))

    def assertMatrixPoint(self, expected_point, matrix, point):
        affine = _to_affine(matrix)
        actual_point = affine * point
        self.assertAlmostEqual(expected_point[0], actual_point[0])
        self.assertAlmostEqual(expected_point[1], actual_point[1])
        return actual_point

    def test_derive(self):
        gm = TestGridMapping(**self.kwargs())
        self.assertEqual((720, 360), gm.size)
        self.assertEqual((360, 180), gm.tile_size)
        self.assertEqual(False, gm.is_j_axis_up)

        # force creating of xy_coords array and save value
        xy_coords = gm.xy_coords

        derived_gm = gm.derive(tile_size=270,
                               is_j_axis_up=True,
                               xy_var_names=('u', 'v'),
                               xy_dim_names=('i', 'j'))

        self.assertIsNot(gm, derived_gm)
        self.assertIsInstance(derived_gm, TestGridMapping)
        self.assertEqual((720, 360), derived_gm.size)
        self.assertEqual((270, 270), derived_gm.tile_size)
        self.assertEqual(True, derived_gm.is_j_axis_up)
        self.assertEqual(('u', 'v'), derived_gm.xy_var_names)
        self.assertEqual(('i', 'j'), derived_gm.xy_dim_names)

        derived_xy_coords = derived_gm.xy_coords
        self.assertIsNot(xy_coords, derived_xy_coords)
        self.assertEqual(((2,), (270, 90), (270, 270, 180)),
                         derived_xy_coords.chunks)

    def test_scale(self):
        gm = TestGridMapping(**self.kwargs())
        self.assertEqual((720, 360), gm.size)
        self.assertEqual((360, 180), gm.tile_size)
        self.assertEqual(False, gm.is_j_axis_up)

        # force creating of xy_coords array and save value
        xy_coords = gm.xy_coords

        scaled_gm = gm.scale((0.25, 0.5))

        self.assertIsNot(gm, scaled_gm)
        self.assertIsInstance(scaled_gm, RegularGridMapping)
        self.assertEqual((180, 180), scaled_gm.size)
        self.assertEqual((180, 180), scaled_gm.tile_size)
        self.assertEqual(False, scaled_gm.is_j_axis_up)
        self.assertEqual(('x', 'y'), scaled_gm.xy_var_names)
        self.assertEqual(('x', 'y'), scaled_gm.xy_dim_names)

        scaled_xy_coords = scaled_gm.xy_coords
        self.assertIsNot(xy_coords, scaled_xy_coords)
        self.assertEqual(((2,), (180,), (180,)),
                         scaled_xy_coords.chunks)

    def test_transform(self):
        gm = TestGridMapping(**self.kwargs(xy_min=(20, 56),
                                           size=(400, 200),
                                           tile_size=(400, 200),
                                           xy_res=(0.01, 0.01)))
        transformed_gm = gm.transform('EPSG:32633')

        self.assertIsNot(gm, transformed_gm)
        self.assertIsInstance(transformed_gm, Coords2DGridMapping)
        self.assertEqual(pyproj.CRS.from_string('EPSG:32633'),
                         transformed_gm.crs)
        self.assertEqual((400, 200), transformed_gm.size)
        self.assertEqual((400, 200), transformed_gm.tile_size)
        self.assertEqual(None, transformed_gm.is_j_axis_up)
        self.assertEqual(('transformed_x', 'transformed_y'),
                         transformed_gm.xy_var_names)
        self.assertEqual(('lon', 'lat'),
                         transformed_gm.xy_dim_names)

    def test_is_close(self):
        gm1 = TestGridMapping(**self.kwargs(xy_min=(0, 0),
                                            size=(400, 200),
                                            xy_res=(0.01, 0.01)))
        gm2 = TestGridMapping(**self.kwargs(xy_min=(0, 0),
                                            size=(400, 200),
                                            xy_res=(0.01, 0.01)))
        self.assertTrue(gm1.is_close(gm1))
        self.assertTrue(gm2.is_close(gm2))
        self.assertTrue(gm1.is_close(gm2))
        self.assertTrue(gm2.is_close(gm1))

        tolerance = 0.001

        gm2 = TestGridMapping(**self.kwargs(xy_min=(tolerance / 2,
                                                    tolerance / 2),
                                            size=(400, 200),
                                            xy_res=(0.01, 0.01)))
        self.assertTrue(gm1.is_close(gm1, tolerance=tolerance))
        self.assertTrue(gm2.is_close(gm2, tolerance=tolerance))
        self.assertTrue(gm1.is_close(gm2, tolerance=tolerance))
        self.assertTrue(gm2.is_close(gm1, tolerance=tolerance))

        gm2 = TestGridMapping(**self.kwargs(xy_min=(tolerance * 2,
                                                    tolerance * 2),
                                            size=(400, 200),
                                            xy_res=(0.01, 0.01)))
        self.assertTrue(gm1.is_close(gm1, tolerance=tolerance))
        self.assertTrue(gm2.is_close(gm2, tolerance=tolerance))
        self.assertFalse(gm1.is_close(gm2, tolerance=tolerance))
        self.assertFalse(gm2.is_close(gm1, tolerance=tolerance))

    def test_ij_bbox_from_xy_bbox(self):
        gm = TestGridMapping(**self.kwargs())

        ij_bbox = gm.ij_bbox_from_xy_bbox((-180, -90, 180, 90))
        self.assertEqual((0, 0, 720, 360), ij_bbox)

        ij_bbox = gm.ij_bbox_from_xy_bbox((-180, -90, 0, 0))
        self.assertEqual((0, 180, 360, 360), ij_bbox)

        ij_bbox = gm.ij_bbox_from_xy_bbox((0, 0, 180, 90))
        self.assertEqual((360, 0, 720, 180), ij_bbox)

        ij_bbox = gm.ij_bbox_from_xy_bbox((-180, -90, 0, 0), ij_border=1)
        self.assertEqual((0, 179, 361, 360), ij_bbox)

        ij_bbox = gm.ij_bbox_from_xy_bbox((0, 0, 180, 90), ij_border=1)
        self.assertEqual((359, 0, 720, 181), ij_bbox)

        ij_bbox = gm.ij_bbox_from_xy_bbox((-190, -100, -170, -80), ij_border=1)
        self.assertEqual((0, 339, 21, 360), ij_bbox)

        ij_bbox = gm.ij_bbox_from_xy_bbox((-190, -100, -180, -90), ij_border=1)
        self.assertEqual((-1, -1, -1, -1), ij_bbox)

    def test_ij_bboxes_from_xy_bboxes(self):
        gm = TestGridMapping(**self.kwargs())

        ij_bboxes = gm.ij_bboxes_from_xy_bboxes(
            xy_bboxes=np.array([
                [-180, -90, 180, 90],
                [-180, -90, 0, 0],
                [0, 0, 180, 90],
                [-180, -90, 0, 0],
                [0, 0, 180, 90],
                [-190, -100, -170, -80],
                [-190, -100, -180, -90]
            ], dtype=np.float32))

        np.testing.assert_equal(
            ij_bboxes,
            np.array([[0, 0, 720, 360],
                      [0, 180, 360, 360],
                      [360, 0, 720, 180],
                      [0, 180, 360, 360],
                      [360, 0, 720, 180],
                      [0, 340, 20, 360],
                      [-1, -1, -1, -1]], dtype=np.int64))

    def test_tile_grid(self):
        gm = GridMapping.regular((7200, 3600),
                                 (-180, -90),
                                 360 / 7200,
                                 'epsg:4326',
                                 tile_size=(720, 360),
                                 is_j_axis_up=True)
        tile_grid = gm.tile_grid
        self.assertIsInstance(tile_grid, TileGrid)
        self.assertEqual([(1, 1),
                          (2, 2),
                          (3, 3),
                          (5, 5),
                          (10, 10)],
                         [tile_grid.get_num_tiles(i)
                          for i in range(tile_grid.num_levels)])
        self.assertEqual([(450, 225),
                          (900, 450),
                          (1800, 900),
                          (3600, 1800),
                          (7200, 3600)],
                         [tile_grid.get_image_size(i)
                          for i in range(tile_grid.num_levels)])
        self.assertEqual([0.8,
                          0.4,
                          0.2,
                          0.1,
                          0.05],
                         [tile_grid.get_resolution(i)
                          for i in range(tile_grid.num_levels)])


# test/core/gridmapping/test_bboxes.py

import unittest

import dask.array as da
import numpy as np
import xarray as xr

from xcube.core.gridmapping.bboxes import compute_ij_bboxes
from xcube.core.gridmapping.bboxes import compute_xy_bbox


class ComputeIJBBoxesTest(unittest.TestCase):

    def setUp(self) -> None:
        lon = xr.DataArray(np.linspace(10., 20., 11), dims='x')
        lat = xr.DataArray(np.linspace(50., 60., 11), dims='y')
        lat, lon = xr.broadcast(lat, lon)
        self.lon_values = lon.values
        self.lat_values = lat.values

    def test_all_included(self):
        xy_bboxes = np.array([[10., 50., 20., 60.]])
        ij_bboxes = np.full_like(xy_bboxes, -1, dtype=np.int64)
        compute_ij_bboxes(self.lon_values, self.lat_values, xy_bboxes, 0.0, 0, ij_bboxes)
        np.testing.assert_almost_equal(ij_bboxes,
                                       np.array([[0, 0, 11, 11]], dtype=np.int64))

    def test_tiles(self):
        a0 = 0.
        a1 = a0 + 5.
        a2 = a1 + 5.
        xy_bboxes = np.array([[10. + a0, 50. + a0, 10. + a1, 50. + a1],
                              [10. + a1, 50. + a0, 10. + a2, 50. + a1],
                              [10. + a0, 50. + a1, 10. + a1, 50. + a2],
                              [10. + a1, 50. + a1, 10. + a2, 50. + a2]])
        ij_bboxes = np.full_like(xy_bboxes, -1, dtype=np.int64)
        compute_ij_bboxes(self.lon_values, self.lat_values, xy_bboxes, 0.0, 0, ij_bboxes)
        np.testing.assert_almost_equal(ij_bboxes,
                                       np.array([[0, 0, 6, 6],
                                                 [5, 0, 11, 6],
                                                 [0, 5, 6, 11],
                                                 [5, 5, 11, 11]], dtype=np.int64))

    def test_none_found(self):
        a0 = 11.
        a1 = a0 + 5.
        a2 = a1 + 5.
        xy_bboxes = np.array([[10. + a0, 50. + a0, 10. + a1, 50. + a1],
                              [10. + a1, 50. + a0, 10. + a2, 50. + a1],
                              [10. + a0, 50. + a1, 10. + a1, 50. + a2],
                              [10. + a1, 50. + a1, 10. + a2, 50. + a2]])
        ij_bboxes = np.full_like(xy_bboxes, -1, dtype=np.int64)
        compute_ij_bboxes(self.lon_values, self.lat_values, xy_bboxes, 0.0, 0, ij_bboxes)
        np.testing.assert_almost_equal(ij_bboxes,
                                       np.array([[-1, -1, -1, -1],
                                                 [-1, -1, -1, -1],
                                                 [-1, -1, -1, -1],
                                                 [-1, -1, -1, -1]], dtype=np.int64))

    def test_with_border(self):
        xy_bboxes = np.array([[12.4, 51.6, 12.6, 51.7]])
        ij_bboxes = np.full_like(xy_bboxes, -1, dtype=np.int64)
        compute_ij_bboxes(self.lon_values, self.lat_values, xy_bboxes, 0.0, 0, ij_bboxes)
        np.testing.assert_almost_equal(ij_bboxes,
                                       np.array([[-1, -1, -1, -1]], dtype=np.int64))
        ij_bboxes = np.full_like(xy_bboxes, -1, dtype=np.int64)
        compute_ij_bboxes(self.lon_values, self.lat_values, xy_bboxes, 0.5, 0, ij_bboxes)
        np.testing.assert_almost_equal(ij_bboxes,
                                       np.array([[2, 2, 4, 3]], dtype=np.int64))
        ij_bboxes = np.full_like(xy_bboxes, -1, dtype=np.int64)
        compute_ij_bboxes(self.lon_values, self.lat_values, xy_bboxes, 1.0, 0, ij_bboxes)
        np.testing.assert_almost_equal(ij_bboxes,
                                       np.array([[2, 1, 4, 3]], dtype=np.int64))
        ij_bboxes = np.full_like(xy_bboxes, -1, dtype=np.int64)
        compute_ij_bboxes(self.lon_values, self.lat_values, xy_bboxes, 2.0, 0, ij_bboxes)
        np.testing.assert_almost_equal(ij_bboxes,
                                       np.array([[1, 0, 5, 4]], dtype=np.int64))
        ij_bboxes = np.full_like(xy_bboxes, -1, dtype=np.int64)
        compute_ij_bboxes(self.lon_values, self.lat_values, xy_bboxes, 2.0, 2, ij_bboxes)
        np.testing.assert_almost_equal(ij_bboxes,
                                       np.array([[0, 0, 7, 6]], dtype=np.int64))


class ComputeXYBBoxTest(unittest.TestCase):
    data = [
        [
            [10, 11, 12, 13, 14],
            [11, 12, 13, 14, 15],
            [12, 13, 14, 15, 16],
            [13, 14, 15, 16, 17],
        ],
        [
            [50, 51, 52, 53, 54],
            [51, 52, 53, 54, 55],
            [52, 53, 54, 55, 56],
            [53, 54, 55, 56, 57],
        ],
    ]

    def test_numpy_array(self):
        xy_coords = np.array(self.data, dtype=np.float64)
        xy_bbox = compute_xy_bbox(xy_coords)
        self.assertEqual((10, 50, 17, 57), xy_bbox)

    def test_dask_array(self):
        xy_coords = da.array(self.data, dtype=np.float64).rechunk((2, 3, 3))
        xy_bbox = compute_xy_bbox(xy_coords)
        self.assertEqual((10, 50, 17, 57), xy_bbox)

    def test_many_nans(self):
        w = 2000
        h = 1000
        x = np.full(h * w, np.nan)
        y = np.full(h * w, np.nan)

        x[np.random.randint(0, w)] = 73.
        y[np.random.randint(0, h)] = 34.

        xy_coords = da.array([x.reshape((h, w)), y.reshape((h, w))], dtype=np.float64) \
            .rechunk((2, 512, 512))
        xy_bbox = compute_xy_bbox(xy_coords)

        self.assertEqual((73., 34., 73., 34.), xy_bbox)


# test/core/gridmapping/test_cfconv.py

import shutil
import unittest
from typing import Set

import fsspec
import numpy as np
import pyproj
import xarray as xr

from test.s3test import MOTO_SERVER_ENDPOINT_URL
from test.s3test import S3Test
from xcube.core.gridmapping import GridMapping
from xcube.core.gridmapping.cfconv import GridCoords
from xcube.core.gridmapping.cfconv import GridMappingProxy
from xcube.core.gridmapping.cfconv import add_spatial_ref
from xcube.core.gridmapping.cfconv import get_dataset_grid_mapping_proxies
from xcube.core.new import new_cube
from xcube.core.store.fs.registry import new_fs_data_store

CRS_WGS84 = pyproj.crs.CRS(4326)
CRS_CRS84 = pyproj.crs.CRS.from_string("urn:ogc:def:crs:OGC:1.3:CRS84")
CRS_UTM_33N = pyproj.crs.CRS(32633)

CRS_ROTATED_POLE = pyproj.crs.CRS.from_cf(
    dict(grid_mapping_name="rotated_latitude_longitude",
         grid_north_pole_latitude=32.5,
         grid_north_pole_longitude=170.))


class GetDatasetGridMappingsTest(unittest.TestCase):
    def test_no_crs_lon_lat_common_names(self):
        dataset = xr.Dataset(coords=dict(lon=xr.DataArray(np.linspace(10, 12, 11), dims='lon'),
                                         lat=xr.DataArray(np.linspace(50, 52, 11), dims='lat')))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn(None, grid_mappings)
        grid_mapping = grid_mappings.get(None)
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual(CRS_WGS84, grid_mapping.crs)
        self.assertEqual('latitude_longitude', grid_mapping.name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('lon', grid_mapping.coords.x.name)
        self.assertEqual('lat', grid_mapping.coords.y.name)

    def test_no_crs_lon_lat_standard_names(self):
        dataset = xr.Dataset(coords=dict(weird_x=xr.DataArray(np.linspace(10, 12, 11), dims='i',
                                                              attrs=dict(standard_name='longitude')),
                                         weird_y=xr.DataArray(np.linspace(50, 52, 11), dims='j',
                                                              attrs=dict(standard_name='latitude'))))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn(None, grid_mappings)
        grid_mapping = grid_mappings.get(None)
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual(CRS_WGS84, grid_mapping.crs)
        self.assertEqual('latitude_longitude', grid_mapping.name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('weird_x', grid_mapping.coords.x.name)
        self.assertEqual('weird_y', grid_mapping.coords.y.name)

    def test_crs_x_y_with_common_names(self):
        dataset = xr.Dataset(dict(crs=xr.DataArray(0, attrs=CRS_UTM_33N.to_cf())),
                             coords=dict(x=xr.DataArray(np.linspace(1000, 12000, 11), dims='x'),
                                         y=xr.DataArray(np.linspace(5000, 52000, 11), dims='y')))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn('crs', grid_mappings)
        grid_mapping = grid_mappings.get('crs')
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual(CRS_UTM_33N, grid_mapping.crs)
        self.assertEqual('transverse_mercator', grid_mapping.name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('x', grid_mapping.coords.x.name)
        self.assertEqual('y', grid_mapping.coords.y.name)

    def test_crs_x_y_with_standard_names(self):
        dataset = xr.Dataset(dict(crs=xr.DataArray(0, attrs=CRS_UTM_33N.to_cf())),
                             coords=dict(myx=xr.DataArray(np.linspace(1000, 12000, 11), dims='x',
                                                          attrs=dict(standard_name='projection_x_coordinate')),
                                         myy=xr.DataArray(np.linspace(5000, 52000, 11), dims='y',
                                                          attrs=dict(standard_name='projection_y_coordinate'))))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn('crs', grid_mappings)
        grid_mapping = grid_mappings.get('crs')
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual(CRS_UTM_33N, grid_mapping.crs)
        self.assertEqual('transverse_mercator', grid_mapping.name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('myx', grid_mapping.coords.x.name)
        self.assertEqual('myy', grid_mapping.coords.y.name)

    def test_rotated_pole_with_common_names(self):
        dataset = xr.Dataset(dict(rotated_pole=xr.DataArray(0, attrs=CRS_ROTATED_POLE.to_cf())),
                             coords=dict(rlon=xr.DataArray(np.linspace(-180, 180, 11), dims='rlon'),
                                         rlat=xr.DataArray(np.linspace(0, 90, 11), dims='rlat')))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn('rotated_pole', grid_mappings)
        grid_mapping = grid_mappings.get('rotated_pole')
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual('Geographic 2D CRS', grid_mapping.crs.type_name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('rlon', grid_mapping.coords.x.name)
        self.assertEqual('rlat', grid_mapping.coords.y.name)

    def test_rotated_pole_with_standard_names(self):
        dataset = xr.Dataset(dict(rotated_pole=xr.DataArray(0, attrs=CRS_ROTATED_POLE.to_cf())),
                             coords=dict(u=xr.DataArray(np.linspace(-180, 180, 11), dims='u',
                                                        attrs=dict(standard_name='grid_longitude')),
                                         v=xr.DataArray(np.linspace(0, 90, 11), dims='v',
                                                        attrs=dict(standard_name='grid_latitude'))))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn('rotated_pole', grid_mappings)
        grid_mapping = grid_mappings.get('rotated_pole')
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual('Geographic 2D CRS', grid_mapping.crs.type_name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('u', grid_mapping.coords.x.name)
        self.assertEqual('v', grid_mapping.coords.y.name)


class XarrayDecodeCfTest(unittest.TestCase):
    """Find out how xarray treats 1D and 2D coordinate variables when decode_cf=True or =False"""

    def test_cf_1d_coords(self):
        self._write_coords(*self._gen_1d())
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=True)
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=False)

    def test_cf_1d_data_vars(self):
        self._write_data_vars(*self._gen_1d())
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=True)
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=False)

    def test_cf_2d_coords(self):
        self._write_coords(*self._gen_2d())
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=True)
        self.assertVarNames({'noise', 'crs', 'lon', 'lat'}, set(), decode_cf=False)

    def test_cf_2d_data_vars(self):
        self._write_data_vars(*self._gen_2d())
        self.assertVarNames({'noise', 'crs', 'lon', 'lat'}, set(), decode_cf=True)
        self.assertVarNames({'noise', 'crs', 'lon', 'lat'}, set(), decode_cf=False)

    def assertVarNames(self, exp_data_vars: Set[str], exp_coords: Set[str], decode_cf: bool):
        ds1 = xr.open_zarr('noise.zarr', decode_cf=decode_cf)
        self.assertEqual(exp_coords, set(ds1.coords))
        self.assertEqual(exp_data_vars, set(ds1.data_vars))

    @classmethod
    def _write_coords(cls, noise, crs, lon, lat):
        dataset = xr.Dataset(dict(noise=noise, crs=crs), coords=dict(lon=lon, lat=lat))
        dataset.to_zarr('noise.zarr', mode='w')

    @classmethod
    def _write_data_vars(cls, noise, crs, lon, lat):
        dataset = xr.Dataset(dict(noise=noise, crs=crs, lon=lon, lat=lat))
        dataset.to_zarr('noise.zarr', mode='w')

    @classmethod
    def tearDownClass(cls) -> None:
        shutil.rmtree('noise.zarr')

    def _gen_1d(self):
        noise = xr.DataArray(np.random.random((11, 11)), dims=('lat', 'lon'))
        crs = xr.DataArray(0, attrs=CRS_CRS84.to_cf())
        lon = xr.DataArray(np.linspace(10, 12, 11), dims='lon')
        lat = xr.DataArray(np.linspace(50, 52, 11), dims='lat')
        noise.attrs['grid_mapping'] = 'crs'
        lon.attrs['standard_name'] = 'longitude'
        lat.attrs['standard_name'] = 'latitude'
        self.assertEqual(('lon',), lon.dims)
        self.assertEqual(('lat',), lat.dims)
        return noise, crs, lon, lat

    def _gen_2d(self):
        noise = xr.DataArray(np.random.random((11, 11)), dims=('y', 'x'))
        crs = xr.DataArray(0, attrs=CRS_CRS84.to_cf())
        lon = xr.DataArray(np.linspace(10, 12, 11), dims='x')
        lat = xr.DataArray(np.linspace(50, 52, 11), dims='y')
        lat, lon = xr.broadcast(lat, lon)
        noise.attrs['grid_mapping'] = 'crs'
        lon.attrs['standard_name'] = 'longitude'
        lat.attrs['standard_name'] = 'latitude'
        self.assertEqual(('y', 'x'), lon.dims)
        self.assertEqual(('y', 'x'), lat.dims)
        return noise, crs, lon, lat


class AddSpatialRefTest(S3Test):

    def test_add_spatial_ref(self):
        self.assert_add_spatial_ref_ok(None, None)
        self.assert_add_spatial_ref_ok(None, ('cx', 'cy'))
        self.assert_add_spatial_ref_ok('crs', None)
        self.assert_add_spatial_ref_ok('crs', ('cx', 'cy'))

    def assert_add_spatial_ref_ok(self, crs_var_name, xy_dim_names):

        root = 'eurodatacube-test/xcube-eea'
        data_id = 'test.zarr'
        crs = pyproj.CRS.from_string("EPSG:3035")

        if xy_dim_names:
            x_name, y_name = xy_dim_names
        else:
            x_name, y_name = 'x', 'y'

        cube = new_cube(x_name=x_name,
                        y_name=y_name,
                        x_start=0,
                        y_start=0,
                        x_res=10,
                        y_res=10,
                        x_units='metres',
                        y_units='metres',
                        drop_bounds=True,
                        width=100,
                        height=100,
                        variables=dict(A=1.3, B=8.3))

        storage_options = dict(
            anon=False,
            client_kwargs=dict(
                endpoint_url=MOTO_SERVER_ENDPOINT_URL,
            )
        )

        fs: fsspec.AbstractFileSystem = fsspec.filesystem('s3',
                                                          **storage_options)
        if fs.isdir(root):
            fs.rm(root, recursive=True)
        fs.mkdirs(root, exist_ok=True)

        data_store = new_fs_data_store('s3',
                                       root=root,
                                       storage_options=storage_options)

        data_store.write_data(cube, data_id=data_id)
        cube = data_store.open_data(data_id)
        self.assertEqual({'A', 'B', 'time', x_name, y_name},
                         set(cube.variables))

        with self.assertRaises(ValueError) as cm:
            GridMapping.from_dataset(cube)
        self.assertEqual(
            ('cannot find any grid mapping in dataset',),
            cm.exception.args
        )

        path = f"{root}/{data_id}"
        group_store = fs.get_mapper(path, create=True)

        expected_crs_var_name = crs_var_name or 'spatial_ref'

        self.assertTrue(fs.exists(path))
        self.assertFalse(fs.exists(f"{path}/{expected_crs_var_name}"))
        self.assertFalse(fs.exists(f"{path}/{expected_crs_var_name}/.zarray"))
        self.assertFalse(fs.exists(f"{path}/{expected_crs_var_name}/.zattrs"))

        kwargs = {}
        if crs_var_name is not None:
            kwargs.update(crs_var_name=crs_var_name)
        if xy_dim_names is not None:
            kwargs.update(xy_dim_names=xy_dim_names)
        add_spatial_ref(group_store, crs, **kwargs)

        self.assertTrue(fs.exists(f"{path}/{expected_crs_var_name}"))
        self.assertTrue(fs.exists(f"{path}/{expected_crs_var_name}/.zarray"))
        self.assertTrue(fs.exists(f"{path}/{expected_crs_var_name}/.zattrs"))

        cube = data_store.open_data(data_id)
        self.assertEqual({'A', 'B', 'time',
                          x_name, y_name, expected_crs_var_name},
                         set(cube.variables))
        self.assertEqual(expected_crs_var_name,
                         cube.A.attrs.get('grid_mapping'))
        self.assertEqual(expected_crs_var_name,
                         cube.B.attrs.get('grid_mapping'))

        gm = GridMapping.from_dataset(cube)
        self.assertIn("LAEA Europe", gm.crs.srs)


# test/core/gridmapping/test_coords.py

import unittest

import dask.array as da
import numpy as np
import pyproj
import xarray as xr

from xcube.core.gridmapping import GridMapping

# noinspection PyProtectedMember

GEO_CRS = pyproj.crs.CRS(4326)


# noinspection PyMethodMayBeStatic
class Coords1DGridMappingTest(unittest.TestCase):

    def test_1d_j_axis_down(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(np.linspace(1.5, 8.5, 8), dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((1, -5, 9, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)

    def test_1d_j_axis_up(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(np.linspace(1.5, 8.5, 8), dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(-4.5, 4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((1, -5, 9, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)

    def test_1d_lon_360(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(np.linspace(177.5, 184.5, 8), dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((177, -5, 185, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_1d_anti_meridian(self):
        lon = np.linspace(177.5, 184.5, 8)
        lon = np.where(lon > 180, lon - 360, lon)
        gm = GridMapping.from_coords(x_coords=xr.DataArray(lon, dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((177, -5, 185, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_1d_tiles_given(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(np.linspace(177.5, 184.5, 8), dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS,
                                     tile_size=(5, 3))
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((5, 3), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((177, -5, 185, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_1d_tiles_from_coords_chunks(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(da.linspace(177.5, 184.5, 8, chunks=4), dims='lon'),
                                     y_coords=xr.DataArray(da.linspace(4.5, -4.5, 10, chunks=5), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((4, 5), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((177, -5, 185, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_1d_x_irregular(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray([1.5, 2.5, 3.5, 4.5, 5.49, 6.5, 7.5, 8.5], dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((1, -5, 9, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)

    def test_1d_xy_coords(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(np.linspace(1.5, 8.5, 8), dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        xy_coords = gm.xy_coords
        self.assertIsInstance(xy_coords, xr.DataArray)
        self.assertIs(xy_coords, gm.xy_coords)
        self.assertEqual(('coord', 'lat', 'lon'), xy_coords.dims)
        self.assertEqual((2, 10, 8), xy_coords.shape)
        self.assertEqual(('lon', 'lat'), gm.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm.xy_dim_names)


class Coords2DGridMappingTest(unittest.TestCase):

    def test_2d(self):
        gm = GridMapping.from_coords(
            x_coords=xr.DataArray([
                [10.0, 10.1, 10.2, 10.3],
                [10.1, 10.2, 10.3, 10.4],
                [10.2, 10.3, 10.4, 10.5],
            ], dims=('lat', 'lon')),
            y_coords=xr.DataArray([
                [52.0, 52.2, 52.4, 52.6],
                [52.2, 52.4, 52.6, 52.8],
                [52.4, 52.6, 52.8, 53.0],
            ], dims=('lat', 'lon')),
            crs=GEO_CRS)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((4, 3), gm.tile_size)
        self.assertEqual((0.3, 0.3), gm.xy_res)
        self.assertEqual((9.85, 51.85, 10.65, 53.15), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)

    def test_2d_tile_size_from_chunks(self):
        gm = GridMapping.from_coords(
            x_coords=xr.DataArray(da.array([
                [10.0, 10.1, 10.2, 10.3],
                [10.1, 10.2, 10.3, 10.4],
                [10.2, 10.3, 10.4, 10.5],
            ]).rechunk((2, 3)), dims=('lat', 'lon')),
            y_coords=xr.DataArray(da.array([
                [52.0, 52.2, 52.4, 52.6],
                [52.2, 52.4, 52.6, 52.8],
                [52.4, 52.6, 52.8, 53.0],
            ]).rechunk((2, 3)), dims=('lat', 'lon')),
            crs=GEO_CRS)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((3, 2), gm.tile_size)

    def test_2d_regular(self):
        gm = GridMapping.from_coords(
            x_coords=xr.DataArray([
                [10.2, 10.3, 10.4, 10.5],
                [10.2, 10.3, 10.4, 10.5],
                [10.2, 10.3, 10.4, 10.5],
            ], dims=('lat', 'lon')),
            y_coords=xr.DataArray([
                [52.4, 52.4, 52.4, 52.4],
                [52.6, 52.6, 52.6, 52.6],
                [52.8, 52.8, 52.8, 52.8],
            ], dims=('lat', 'lon')),
            crs=GEO_CRS)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((4, 3), gm.tile_size)
        self.assertAlmostEqual(0.1, gm.x_res)
        self.assertAlmostEqual(0.2, gm.y_res)
        self.assertAlmostEqual(10.15, gm.x_min)
        self.assertAlmostEqual(52.3, gm.y_min)
        self.assertAlmostEqual(10.55, gm.x_max)
        self.assertAlmostEqual(52.9, gm.y_max)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)

    def test_2d_anti_meridian(self):
        gm = GridMapping.from_coords(
            x_coords=xr.DataArray([
                [+177.5, +178.5, +179.5, -179.5],
                [+178.5, +179.5, -179.5, -178.5],
                [+179.5, -179.5, -178.5, -177.5],
            ], dims=('lat', 'lon')),
            y_coords=xr.DataArray([
                [52.4, 52.4, 52.4, 52.4],
                [52.6, 52.6, 52.6, 52.6],
                [52.8, 52.8, 52.8, 52.8],
            ], dims=('lat', 'lon')),
            crs=GEO_CRS)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((4, 3), gm.tile_size)
        self.assertAlmostEqual(0.2, gm.x_res)
        self.assertAlmostEqual(0.2, gm.y_res)
        self.assertEqual((177.4, 52.3, 182.6, 52.9), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_to_regular(self):
        lon = xr.DataArray([[1.0, 6.0],
                            [0.0, 2.0]], dims=('y', 'x'))
        lat = xr.DataArray([[56.0, 53.0],
                            [52.0, 50.0]], dims=('y', 'x'))

        gm_irr = GridMapping.from_coords(lon, lat, GEO_CRS)
        gm_reg_actual = gm_irr.to_regular()
        gm_reg_expected = GridMapping.regular(size=(4, 4),
                                              xy_min=(-2, 48),
                                              xy_res=4.0,
                                              crs=GEO_CRS)
        self.assertEqual(gm_reg_expected.size, gm_reg_actual.size)
        self.assertEqual(gm_reg_expected.tile_size, gm_reg_actual.tile_size)
        self.assertEqual(gm_reg_expected.xy_res, gm_reg_actual.xy_res)
        self.assertEqual(gm_reg_expected.xy_bbox, gm_reg_actual.xy_bbox)
        self.assertEqual(gm_reg_expected.crs, gm_reg_actual.crs)

    def test_2d_xy_coords(self):
        gm = GridMapping.from_coords(
            x_coords=xr.DataArray([
                [10.0, 10.1, 10.2, 10.3],
                [10.1, 10.2, 10.3, 10.4],
                [10.2, 10.3, 10.4, 10.5],
            ], dims=('lat', 'lon')),
            y_coords=xr.DataArray([
                [52.0, 52.2, 52.4, 52.6],
                [52.2, 52.4, 52.6, 52.8],
                [52.4, 52.6, 52.8, 53.0],
            ], dims=('lat', 'lon')),
            crs=GEO_CRS)
        xy_coords = gm.xy_coords
        self.assertIsInstance(xy_coords, xr.DataArray)
        self.assertIs(xy_coords, gm.xy_coords)
        self.assertEqual(('coord', 'lat', 'lon'), xy_coords.dims)
        self.assertEqual((2, 3, 4), xy_coords.shape)
        self.assertEqual(('lon', 'lat'), gm.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm.xy_dim_names)


# test/core/gridmapping/test_dataset.py

import os.path
import unittest

import numpy as np
import pyproj
import xarray as xr

import xcube.core.new
from test.sampledata import create_s2plus_dataset
from xcube.core.gridmapping import GridMapping

# noinspection PyProtectedMember

GEO_CRS = pyproj.crs.CRS(4326)
NOT_A_GEO_CRS = pyproj.crs.CRS(5243)


# noinspection PyMethodMayBeStatic
class DatasetGridMappingTest(unittest.TestCase):
    def test_from_regular_cube(self):
        dataset = xcube.core.new.new_cube(variables=dict(rad=0.5))
        gm = GridMapping.from_dataset(dataset)
        self.assertEqual((360, 180), gm.size)
        self.assertEqual((360, 180), gm.tile_size)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_lon_360)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual((2, 180, 360), gm.xy_coords.shape)
        self.assertEqual(('coord', 'lat', 'lon'), gm.xy_coords.dims)

    def test_from_regular_cube_with_crs(self):
        dataset = xcube.core.new.new_cube(variables=dict(rad=0.5),
                                          x_start=0,
                                          y_start=0,
                                          x_name='x',
                                          y_name='y',
                                          crs='epsg:25832')
        gm1 = GridMapping.from_dataset(dataset)
        self.assertEqual(pyproj.CRS.from_string('epsg:25832'), gm1.crs)
        dataset = dataset.drop_vars('crs')
        gm2 = GridMapping.from_dataset(dataset)
        self.assertEqual(GEO_CRS, gm2.crs)
        gm3 = GridMapping.from_dataset(dataset, crs=gm1.crs)
        self.assertEqual(gm1.crs, gm3.crs)
        self.assertEqual(('x', 'y'), gm3.xy_var_names)
        self.assertEqual(('x', 'y'), gm3.xy_dim_names)

    def test_from_regular_cube_no_chunks_and_chunks(self):
        dataset = xcube.core.new.new_cube(variables=dict(rad=0.5))
        gm1 = GridMapping.from_dataset(dataset)
        self.assertEqual((360, 180), gm1.tile_size)
        dataset = dataset.chunk(dict(lon=10, lat=20))
        gm2 = GridMapping.from_dataset(dataset)
        self.assertEqual((10, 20), gm2.tile_size)

    def test_from_non_regular_cube(self):
        lon = np.array([[8, 9.3, 10.6, 11.9],
                        [8, 9.2, 10.4, 11.6],
                        [8, 9.1, 10.2, 11.3]], dtype=np.float32)
        lat = np.array([[56, 56.1, 56.2, 56.3],
                        [55, 55.2, 55.4, 55.6],
                        [54, 54.3, 54.6, 54.9]], dtype=np.float32)
        rad = np.random.random(3 * 4).reshape((3, 4))
        dims = ('y', 'x')
        dataset = xr.Dataset(dict(lon=xr.DataArray(lon, dims=dims),
                                  lat=xr.DataArray(lat, dims=dims),
                                  rad=xr.DataArray(rad, dims=dims)))
        gm = GridMapping.from_dataset(dataset)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((4, 3), gm.tile_size)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(False, gm.is_lon_360)
        self.assertEqual(None, gm.is_j_axis_up)
        self.assertEqual((2, 3, 4), gm.xy_coords.shape)
        self.assertEqual(('coord', 'y', 'x'), gm.xy_coords.dims)
        self.assertEqual((0.8, 0.8), gm.xy_res)

    def test_from_real_olci(self):
        olci_l2_path = os.path.join(os.path.dirname(__file__),
                                    '..', '..', '..',
                                    'examples', 'notebooks', 'inputdata',
                                    'S3-OLCI-L2A.zarr.zip')

        dataset = xr.open_zarr(olci_l2_path)
        gm = GridMapping.from_dataset(dataset)
        self.assertEqual((1189, 1890), gm.size)
        self.assertEqual((512, 512), gm.tile_size)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual((0.0025, 0.0025), gm.xy_res)
        # self.assertAlmostEqual(12.693771178309552, gm.x_min)
        # self.assertAlmostEqual(20.005413821690446, gm.x_max)
        # self.assertAlmostEqual(55.19965017830955, gm.y_min)
        # self.assertAlmostEqual(60.63871982169044, gm.y_max)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(False, gm.is_lon_360)
        self.assertEqual(None, gm.is_j_axis_up)
        self.assertEqual((2, 1890, 1189), gm.xy_coords.shape)
        self.assertEqual(('coord', 'y', 'x'), gm.xy_coords.dims)

        gm = gm.to_regular()
        self.assertEqual((2926, 2177), gm.size)

    def test_from_sentinel_2(self):
        dataset = create_s2plus_dataset()

        gm = GridMapping.from_dataset(dataset)
        # Should pick the projected one which is regular
        self.assertEqual("Projected CRS", gm.crs.type_name)
        self.assertEqual(True, gm.is_regular)

        gm = GridMapping.from_dataset(dataset, prefer_is_regular=True)
        # Should pick the projected one which is regular
        self.assertEqual("Projected CRS", gm.crs.type_name)
        self.assertEqual(True, gm.is_regular)

        gm = GridMapping.from_dataset(dataset, prefer_is_regular=False)
        # Should pick the geographic one which is irregular
        self.assertEqual('Geographic 2D CRS', gm.crs.type_name)
        self.assertEqual(False, gm.is_regular)

        gm = GridMapping.from_dataset(dataset, prefer_crs=GEO_CRS)
        # Should pick the geographic one which is irregular
        self.assertEqual('Geographic 2D CRS', gm.crs.type_name)
        self.assertEqual(False, gm.is_regular)

        gm = GridMapping.from_dataset(dataset, prefer_crs=GEO_CRS, prefer_is_regular=True)
        # Should pick the geographic one which is irregular
        self.assertEqual('Geographic 2D CRS', gm.crs.type_name)
        self.assertEqual(False, gm.is_regular)

    def test_no_grid_mapping_found(self):
        with self.assertRaises(ValueError) as cm:
            GridMapping.from_dataset(xr.Dataset())
        self.assertEqual('cannot find any grid mapping in dataset', f'{cm.exception}')


# test/core/gridmapping/test_helpers.py

import unittest
from fractions import Fraction

from xcube.core.gridmapping.helpers import _to_int_or_float
from xcube.core.gridmapping.helpers import round_to_fraction


class RoundToFractionTest(unittest.TestCase):
    dump = False

    def test_invalid(self):
        with self.assertRaises(ValueError):
            round_to_fraction(0.29, digits=0)
        with self.assertRaises(ValueError):
            round_to_fraction(0.29, resolution=0)
        with self.assertRaises(ValueError):
            round_to_fraction(0.29, resolution=0.12)

    def test_1_025(self):
        def f(value):
            return float(round_to_fraction(value, 1, 0.25))

        self.assertAlmostEqual(-1, f(-1))
        self.assertAlmostEqual(0.0, f(0))
        self.assertAlmostEqual(1.0, f(1))
        self.assertAlmostEqual(1.25, f(1.2))
        self.assertAlmostEqual(1.25, f(1.3))
        self.assertAlmostEqual(1.5, f(1.4))
        self.assertAlmostEqual(1.5, f(1.45))
        self.assertAlmostEqual(1.5, f(1.51))
        self.assertAlmostEqual(1.75, f(1.7))
        self.assertAlmostEqual(2.0, f(1.9))
        self.assertAlmostEqual(2.0, f(1.96))
        self.assertAlmostEqual(2.0, f(1.98))
        self.assertAlmostEqual(2.0, f(2))

    def test_2_025(self):
        def f(value):
            return float(round_to_fraction(value, 2, 0.25))

        self.assertAlmostEqual(-1, f(-1))
        self.assertAlmostEqual(0.0, f(0))
        self.assertAlmostEqual(1.0, f(1))
        self.assertAlmostEqual(1.2, f(1.2))
        self.assertAlmostEqual(1.225, f(1.23))
        self.assertAlmostEqual(1.3, f(1.3))
        self.assertAlmostEqual(1.4, f(1.4))
        self.assertAlmostEqual(1.45, f(1.45))
        self.assertAlmostEqual(1.5, f(1.51))
        self.assertAlmostEqual(1.7, f(1.7))
        self.assertAlmostEqual(1.8, f(1.79))
        self.assertAlmostEqual(1.9, f(1.9))
        self.assertAlmostEqual(1.95, f(1.96))
        self.assertAlmostEqual(1.975, f(1.98))
        self.assertAlmostEqual(2.0, f(2))

    def test_default(self):
        values = [
            [-1.0, -1.0, Fraction(-1, 1)],
            [0.0, 0.0, Fraction(0, 1)],
            [5.247476065426347e-09, 5.2e-09, Fraction(13, 2500000000)],
            [3.427467229408875e-06, 3.4e-06, Fraction(17, 5000000)],
            [4.501758583626108e-06, 4.5e-06, Fraction(9, 2000000)],
            [1.1351705264714663e-05, 1.1e-05, Fraction(11, 1000000)],
            [0.00048171747406886744, 0.00048, Fraction(3, 6250)],
            [0.0018032657496927416, 0.0018, Fraction(9, 5000)],
            [0.0019897341919324425, 0.002, Fraction(1, 500)],
            [0.0041643509375105065, 0.0042, Fraction(21, 5000)],
            [0.030607346091352187, 0.031, Fraction(31, 1000)],
            [1.0076973439575128, 1.0, Fraction(1, 1)],
            [1.0, 1.0, Fraction(1, 1)],
            [84.54360269093455, 85.0, Fraction(85, 1)],
            [494.86581234602096, 490.0, Fraction(490, 1)],
            [987.9441243998718, 990.0, Fraction(990, 1)],
            [1757.368043916636, 1800.0, Fraction(1800, 1)],
            [1143506.2928512183, 1100000.0, Fraction(1100000, 1)],
            [217971970.75235566, 220000000.0, Fraction(220000000, 1)]
        ]
        self._assert_values(values, dict())

    def test_3_025(self):
        actual = round_to_fraction(1, digits=1, resolution=0.25)
        self.assertEqual(Fraction(1, 1), actual)
        values = [
            [-1.0, -1.0, Fraction(-1, 1)],
            [0.0, 0.0, Fraction(0, 1)],
            [5.247476065426347e-09, 5.2475e-09, Fraction(2099, 400000000000)],
            [3.427467229408875e-06, 3.4275e-06, Fraction(1371, 400000000)],
            [4.501758583626108e-06, 4.5025e-06, Fraction(1801, 400000000)],
            [1.1351705264714663e-05, 1.135e-05, Fraction(227, 20000000)],
            [0.00048171747406886744, 0.00048175, Fraction(1927, 4000000)],
            [0.0018032657496927416, 0.0018025, Fraction(721, 400000)],
            [0.0019897341919324425, 0.00199, Fraction(199, 100000)],
            [0.0041643509375105065, 0.004165, Fraction(833, 200000)],
            [0.030607346091352187, 0.0306, Fraction(153, 5000)],
            [1.0076973439575128, 1.0075, Fraction(403, 400)],
            [1.0, 1.0, Fraction(1, 1)],
            [84.54360269093455, 84.55, Fraction(1691, 20)],
            [494.86581234602096, 494.75, Fraction(1979, 4)],
            [987.9441243998718, 988.0, Fraction(988, 1)],
            [1757.368043916636, 1757.5, Fraction(3515, 2)],
            [1143506.2928512183, 1142500.0, Fraction(1142500, 1)],
            [217971970.75235566, 218000000.0, Fraction(218000000, 1)]
        ]
        self._assert_values(values, dict(digits=3, resolution=0.25))

    def test_2_5(self):
        values = [
            [-1.0, -1.0, Fraction(-1, 1)],
            [0.0, 0.0, Fraction(0, 1)],
            [5.247476065426347e-09, 5.25e-09, Fraction(21, 4000000000)],
            [3.427467229408875e-06, 3.45e-06, Fraction(69, 20000000)],
            [4.501758583626108e-06, 4.5e-06, Fraction(9, 2000000)],
            [1.1351705264714663e-05, 1.15e-05, Fraction(23, 2000000)],
            [0.00048171747406886744, 0.00048, Fraction(3, 6250)],
            [0.0018032657496927416, 0.0018, Fraction(9, 5000)],
            [0.0019897341919324425, 0.002, Fraction(1, 500)],
            [0.0041643509375105065, 0.00415, Fraction(83, 20000)],
            [0.030607346091352187, 0.0305, Fraction(61, 2000)],
            [1.0076973439575128, 1.0, Fraction(1, 1)],
            [1.0, 1.0, Fraction(1, 1)],
            [84.54360269093455, 84.5, Fraction(169, 2)],
            [494.86581234602096, 495.0, Fraction(495, 1)],
            [987.9441243998718, 990.0, Fraction(990, 1)],
            [1757.368043916636, 1750.0, Fraction(1750, 1)],
            [1143506.2928512183, 1150000.0, Fraction(1150000, 1)],
            [217971970.75235566, 220000000.0, Fraction(220000000, 1)]
        ]
        self._assert_values(values, dict(digits=2, resolution=0.5))

    def _assert_values(self, values, kwargs):
        if self.dump:
            results = []
            for value, _, _ in values:
                actual_fraction = round_to_fraction(value, **kwargs)
                results.append([value, float(actual_fraction), actual_fraction])
            import pprint
            pprint.pprint(results)

        for value, expected_float, expected_fraction in values:
            actual_fraction = round_to_fraction(value, **kwargs)
            self.assertEqual(expected_fraction, actual_fraction)
            self.assertAlmostEqual(expected_float, float(actual_fraction))


class ToIntOrFloatTest(unittest.TestCase):

    def test_down_to_int(self):
        result = _to_int_or_float(90.0001)
        self.assertEqual(90, result)

    def test_leave_as_bigger_float(self):
        result = _to_int_or_float(90.001)
        self.assertEqual(90.001, result)

    def test_up_to_int(self):
        result = _to_int_or_float(89.9999)
        self.assertEqual(90, result)

    def test_leave_as_smaller_float(self):
        result = _to_int_or_float(89.999)
        self.assertEqual(89.999, result)

    def test_up_to_int_small_value(self):
        result = _to_int_or_float(0.99999)
        self.assertEqual(1, result)

    def test_leave_as_smaller_float_small_value(self):
        result = _to_int_or_float(0.9999)
        self.assertEqual(0.9999, result)


# test/core/gridmapping/test_regular.py

import unittest

import numpy as np
import pyproj
import xarray as xr

from xcube.core.gridmapping import CRS_WGS84
from xcube.core.gridmapping import GridMapping
from xcube.core.gridmapping.regular import RegularGridMapping

# noinspection PyProtectedMember

GEO_CRS = pyproj.crs.CRS(4326)
NOT_A_GEO_CRS = pyproj.crs.CRS(5243)


# noinspection PyMethodMayBeStatic
class RegularGridMappingTest(unittest.TestCase):

    def test_default_props(self):
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, CRS_WGS84)
        self.assertEqual((1000, 1000), gm.size)
        self.assertEqual((1000, 1000), gm.tile_size)
        self.assertEqual(10, gm.x_min)
        self.assertEqual(53, gm.y_min)
        self.assertEqual((0.01, 0.01), gm.xy_res)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)

    def test_invalid_y(self):
        with self.assertRaises(ValueError) as cm:
            GridMapping.regular((1000, 1000), (10, -90.5), 0.01, CRS_WGS84)
        self.assertEqual('invalid y_min', f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            GridMapping.regular((1000, 1000), (10, 53), 0.1, CRS_WGS84)
        self.assertEqual('invalid size, y_min combination', f'{cm.exception}')

    def test_xy_bbox(self):
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, CRS_WGS84)
        self.assertEqual((10, 53, 20, 63), gm.xy_bbox)
        self.assertEqual(False, gm.is_lon_360)

    def test_xy_bbox_anti_meridian(self):
        gm = GridMapping.regular((2000, 1000), (174.0, -30.0), 0.005, CRS_WGS84)
        self.assertEqual((174.0, -30.0, 184.0, -25.0), gm.xy_bbox)
        self.assertEqual(True, gm.is_lon_360)

    def test_derive(self):
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, CRS_WGS84)
        self.assertEqual((1000, 1000), gm.size)
        self.assertEqual((1000, 1000), gm.tile_size)
        self.assertEqual(False, gm.is_j_axis_up)
        derived_gm = gm.derive(tile_size=500, is_j_axis_up=True)
        self.assertIsNot(gm, derived_gm)
        self.assertIsInstance(derived_gm, RegularGridMapping)
        self.assertEqual((1000, 1000), derived_gm.size)
        self.assertEqual((500, 500), derived_gm.tile_size)
        self.assertEqual(True, derived_gm.is_j_axis_up)

    def test_xy_coords(self):
        gm = GridMapping.regular((8, 4), (10, 53), 0.1, CRS_WGS84).derive(tile_size=(4, 2))
        xy_coords = gm.xy_coords
        self.assertIsInstance(xy_coords, xr.DataArray)
        self.assertIs(gm.xy_coords, xy_coords)
        self.assertEqual(('coord', 'lat', 'lon'), xy_coords.dims)
        self.assertEqual((2, 4, 8), xy_coords.shape)
        self.assertEqual(((2,), (2, 2), (4, 4)), xy_coords.chunks)
        np.testing.assert_almost_equal(
            np.array([
                [10.05, 10.15, 10.25, 10.35, 10.45, 10.55, 10.65, 10.75],
                [10.05, 10.15, 10.25, 10.35, 10.45, 10.55, 10.65, 10.75],
                [10.05, 10.15, 10.25, 10.35, 10.45, 10.55, 10.65, 10.75],
                [10.05, 10.15, 10.25, 10.35, 10.45, 10.55, 10.65, 10.75]
            ]),
            xy_coords.values[0]
        )
        np.testing.assert_almost_equal(
            np.array([
                [53.35, 53.35, 53.35, 53.35, 53.35, 53.35, 53.35, 53.35],
                [53.25, 53.25, 53.25, 53.25, 53.25, 53.25, 53.25, 53.25],
                [53.15, 53.15, 53.15, 53.15, 53.15, 53.15, 53.15, 53.15],
                [53.05, 53.05, 53.05, 53.05, 53.05, 53.05, 53.05, 53.05]
            ]),
            xy_coords.values[1]
        )

    def test_xy_names(self):
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, GEO_CRS).derive(tile_size=500)
        self.assertEqual(('lon', 'lat'), gm.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm.xy_dim_names)
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, NOT_A_GEO_CRS).derive(tile_size=500)
        self.assertEqual(('x', 'y'), gm.xy_var_names)
        self.assertEqual(('x', 'y'), gm.xy_dim_names)

    def test_ij_bboxes(self):
        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS)
        np.testing.assert_almost_equal(gm.ij_bboxes,
                                       np.array([[0, 0, 2000, 1000]],
                                                dtype=np.int64))

        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS).derive(tile_size=500)
        np.testing.assert_almost_equal(gm.ij_bboxes,
                                       np.array([
                                           [0, 0, 500, 500],
                                           [500, 0, 1000, 500],
                                           [1000, 0, 1500, 500],
                                           [1500, 0, 2000, 500],
                                           [0, 500, 500, 1000],
                                           [500, 500, 1000, 1000],
                                           [1000, 500, 1500, 1000],
                                           [1500, 500, 2000, 1000]
                                       ], dtype=np.int64))

    def test_xy_bboxes(self):
        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS)
        np.testing.assert_almost_equal(gm.xy_bboxes,
                                       np.array([[10., 20., 210., 120.]],
                                                dtype=np.float64))

        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS).derive(tile_size=500)
        np.testing.assert_almost_equal(gm.xy_bboxes,
                                       np.array([
                                           [10., 70, 60, 120.],
                                           [60., 70, 110, 120.],
                                           [110., 70, 160, 120.],
                                           [160., 70, 210, 120.],
                                           [10., 20, 60, 70.],
                                           [60., 20, 110, 70.],
                                           [110., 20, 160, 70.],
                                           [160., 20, 210, 70.]
                                       ], dtype=np.float64))

    def test_xy_bboxes_is_j_axis_up(self):
        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS).derive(is_j_axis_up=True)
        np.testing.assert_almost_equal(gm.xy_bboxes,
                                       np.array([[10., 20., 210., 120.]],
                                                dtype=np.float64))

        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS, ).derive(tile_size=500,
                                                             is_j_axis_up=True)
        np.testing.assert_almost_equal(gm.xy_bboxes,
                                       np.array([
                                           [10., 20., 60., 70.],
                                           [60., 20., 110., 70.],
                                           [110., 20., 160., 70.],
                                           [160., 20., 210., 70.],
                                           [10., 70., 60., 120.],
                                           [60., 70., 110., 120.],
                                           [110., 70., 160., 120.],
                                           [160., 70., 210., 120.]
                                       ], dtype=np.float64))

    def test_to_coords(self):
        gm = GridMapping.regular(size=(10, 6),
                                 xy_min=(-2600.0, 1200.0),
                                 xy_res=10.0,
                                 crs=NOT_A_GEO_CRS)

        cv = gm.to_coords(xy_var_names=('x', 'y'))
        self._assert_coord_vars(cv,
                                (10, 6),
                                ('x', 'y'),
                                (-2595., -2505.),
                                (1255., 1205.),
                                ('x_bnds', 'y_bnds'),
                                (
                                    (-2600., -2590.),
                                    (-2510., -2500.),
                                ),
                                (
                                    (1260., 1250.),
                                    (1210., 1200.),
                                ))

    def test_coord_vars_j_axis_up(self):
        gm = GridMapping.regular(size=(10, 6),
                                 xy_min=(-2600.0, 1200.0),
                                 xy_res=10.0,
                                 crs=NOT_A_GEO_CRS).derive(is_j_axis_up=True)

        cv = gm.to_coords(xy_var_names=('x', 'y'))
        self._assert_coord_vars(cv,
                                (10, 6),
                                ('x', 'y'),
                                (-2595., -2505.),
                                (1205., 1255.),
                                ('x_bnds', 'y_bnds'),
                                (
                                    (-2600., -2590.),
                                    (-2510., -2500.),
                                ),
                                (
                                    (1200., 1210.),
                                    (1250., 1260.),
                                ))

    def test_coord_vars_antimeridian(self):
        gm = GridMapping.regular(size=(10, 10),
                                 xy_min=(172.0, 53.0),
                                 xy_res=2.0,
                                 crs=GEO_CRS)

        cv = gm.to_coords(xy_var_names=('lon', 'lat'))
        self._assert_coord_vars(cv,
                                (10, 10),
                                ('lon', 'lat'),
                                (173.0, -169.0),
                                (72.0, 54.0),
                                ('lon_bnds', 'lat_bnds'),
                                (
                                    (172., 174.),
                                    (-170., -168.),
                                ),
                                (
                                    (73., 71.),
                                    (55., 53.),
                                ))

    def _assert_coord_vars(self,
                           cv,
                           size,
                           xy_names,
                           x_values,
                           y_values,
                           xy_bnds_names,
                           x_bnds_values,
                           y_bnds_values):
        self.assertIsNotNone(cv)
        self.assertIn(xy_names[0], cv)
        self.assertIn(xy_names[1], cv)
        self.assertIn(xy_bnds_names[0], cv)
        self.assertIn(xy_bnds_names[1], cv)

        x = cv[xy_names[0]]
        self.assertEqual((size[0],), x.shape)
        np.testing.assert_almost_equal(x.values[0], np.array(x_values[0]))
        np.testing.assert_almost_equal(x.values[-1], np.array(x_values[-1]))

        y = cv[xy_names[1]]
        self.assertEqual((size[1],), y.shape)
        np.testing.assert_almost_equal(y.values[0], np.array(y_values[0]))
        np.testing.assert_almost_equal(y.values[-1], np.array(y_values[-1]))

        x_bnds = cv[xy_bnds_names[0]]
        self.assertEqual((size[0], 2), x_bnds.shape)
        np.testing.assert_almost_equal(x_bnds.values[0], np.array(x_bnds_values[0]))
        np.testing.assert_almost_equal(x_bnds.values[-1], np.array(x_bnds_values[-1]))

        y_bnds = cv[xy_bnds_names[1]]
        self.assertEqual((size[1], 2), y_bnds.shape)
        np.testing.assert_almost_equal(y_bnds.values[0], y_bnds_values[0])
        np.testing.assert_almost_equal(y_bnds.values[-1], y_bnds_values[-1])


# test/core/gridmapping/test_transform.py

import unittest

import pyproj.crs

from test.sampledata import create_s2plus_dataset
from xcube.core.gridmapping import CRS_CRS84
from xcube.core.gridmapping import CRS_WGS84
from xcube.core.gridmapping import GridMapping

CRS_UTM_32N = pyproj.crs.CRS(32632)

import numpy as np


# noinspection PyMethodMayBeStatic
class TransformTest(unittest.TestCase):

    def test_transform(self):
        gm = GridMapping.regular(size=(3, 3), xy_min=(10, 53), xy_res=0.1, crs=CRS_CRS84)
        gm_t = gm.transform(crs=CRS_UTM_32N)

        self.assertEqual(CRS_UTM_32N, gm_t.crs)
        self.assertEqual(False, gm_t.is_regular)
        self.assertEqual(('transformed_x', 'transformed_y'), gm_t.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm_t.xy_dim_names)
        np.testing.assert_almost_equal(
            np.array([
                [570057.076286, 576728.9360228, 583400.7295284],
                [570220.3304187, 576907.7404859, 583595.0849538],
                [570383.3684844, 577086.3083212, 583789.1831954]
            ]),
            gm_t.xy_coords[0])
        np.testing.assert_almost_equal(
            np.array([
                [5900595.928991, 5900698.5746648, 5900810.5532744],
                [5889471.9033896, 5889574.6540572, 5889686.7472201],
                [5878348.0594403, 5878450.9138481, 5878563.1201969]
            ]),
            gm_t.xy_coords[1])

    def test_transform_xy_var_names(self):
        gm = GridMapping.regular(size=(3, 3), xy_min=(10, 53), xy_res=0.1, crs=CRS_CRS84)
        gm_t = gm.transform(crs=CRS_UTM_32N, xy_var_names=('x', 'y'))
        self.assertEqual(CRS_UTM_32N, gm_t.crs)
        self.assertEqual(('x', 'y'), gm_t.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm_t.xy_dim_names)

    def test_transform_s2(self):
        dataset = create_s2plus_dataset()

        gm = GridMapping.from_dataset(dataset, prefer_is_regular=True)
        # Assert we've picked the projected one which is regular
        self.assertEqual("Projected CRS", gm.crs.type_name)
        self.assertEqual(True, gm.is_regular)

        gm_t = gm.transform(CRS_CRS84)
        self.assertEqual(CRS_CRS84, gm_t.crs)

        gm_t = gm.transform(CRS_WGS84)
        self.assertEqual(CRS_WGS84, gm_t.crs)

    def test_transform_no_op(self):
        dataset = create_s2plus_dataset()

        gm = GridMapping.from_dataset(dataset, prefer_is_regular=True)
        # Assert we've picked the projected one which is regular
        self.assertEqual("Projected CRS", gm.crs.type_name)
        self.assertEqual(True, gm.is_regular)

        gm_t = gm.transform(gm.crs)
        self.assertIs(gm, gm_t)

        # Almost no op
        gm = GridMapping.regular(size=(3, 3), xy_min=(10, 53), xy_res=0.1, crs=CRS_CRS84)
        gm_t = gm.transform(crs=gm.crs, xy_var_names=('x', 'y'))
        self.assertEqual(('x', 'y'), gm_t.xy_var_names)



# test/core/resampling/test_affine.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.gridmapping import GridMapping
from xcube.core.resampling import affine_transform_dataset

nan = np.nan

res = 0.1

source_ds = xr.Dataset(
    data_vars=dict(
        refl=xr.DataArray(
            np.array([
                [0, 1, 0, 2, 0, 3, 0, 4],
                [2, 0, 3, 0, 4, 0, 1, 0],
                [0, 4, 0, 1, 0, 2, 0, 3],
                [1, 0, 2, 0, 3, 0, 4, 0],
                [0, 3, 0, 4, 0, 1, 0, 2],
                [4, 0, 1, 0, 2, 0, 3, 0],
            ], dtype=np.float64),
            dims=('lat', 'lon'))
    ),
    coords=dict(
        lon=xr.DataArray(50.0 + res * np.arange(0, 8) + 0.5 * res, dims='lon'),
        lat=xr.DataArray(10.6 - res * np.arange(0, 6) - 0.5 * res, dims='lat')
    )
)

source_gm = GridMapping.from_dataset(source_ds)


class AffineTransformDatasetTest(unittest.TestCase):
    def test_subset(self):
        target_gm = GridMapping.regular((3, 3), (50.0, 10.0), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((3, 3), target_ds.refl.shape)
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [1, 0, 2],
                [0, 3, 0],
                [4, 0, 1],
            ]))

        target_gm = GridMapping.regular((3, 3), (50.1, 10.1), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((3, 3), target_ds.refl.shape)
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [4, 0, 1],
                [0, 2, 0],
                [3, 0, 4],
            ]))

        target_gm = GridMapping.regular((3, 3), (50.05, 10.05), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((3, 3), target_ds.refl.shape)
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [1.25, 1.5, 0.75],
                [1., 1.25, 1.5],
                [1.75, 1., 1.25]
            ]))

    def test_downscale_x2(self):
        target_gm = GridMapping.regular((8, 6), (50, 10), 2 * res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [0.75, 1.25, 1.75, 1.25, nan, nan, nan, nan],
                [1.25, 0.75, 1.25, 1.75, nan, nan, nan, nan],
                [1.75, 1.25, 0.75, 1.25, nan, nan, nan, nan]
            ]))

    def test_downscale_x2_and_shift(self):
        target_gm = GridMapping.regular((8, 6), (49.8, 9.8), 2 * res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, 0.75, 1.25, 1.75, 1.25, nan, nan, nan],
                [nan, 1.25, 0.75, 1.25, 1.75, nan, nan, nan],
                [nan, 1.75, 1.25, 0.75, 1.25, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan]
            ]))

    def test_upscale_x2(self):
        target_gm = GridMapping.regular((8, 6), (50, 10), res / 2, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [1.0, 0.5, 0.0, 1.0, 2.0, 1.0, 0.0, 1.5],
                [0.5, 1.0, 1.5, 1.25, 1.0, 1.5, 2.0, 1.75],
                [0.0, 1.5, 3.0, 1.5, 0.0, 2.0, 4.0, 2.0],
                [2.0, 1.75, 1.5, 1.0, 0.5, 1.25, 2.0, 1.5],
                [4.0, 2.0, 0.0, 0.5, 1.0, 0.5, 0.0, 1.0],
                [nan, nan, nan, nan, nan, nan, nan, nan]
            ]))

    def test_upscale_x2_and_shift(self):
        target_gm = GridMapping.regular((8, 6), (49.9, 9.95), res / 2, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, 0.5, 1.0, 1.5, 1.25, 1.0, 1.5],
                [nan, nan, 0.0, 1.5, 3.0, 1.5, 0.0, 2.0],
                [nan, nan, 2.0, 1.75, 1.5, 1.0, 0.5, 1.25],
                [nan, nan, 4.0, 2.0, 0.0, 0.5, 1.0, 0.5],
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan]
            ]))

    def test_shift(self):
        target_gm = GridMapping.regular((8, 6), (50.2, 10.1), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [0.0, 2.0, 0.0, 3.0, 0.0, 4.0, nan, nan],
                [3.0, 0.0, 4.0, 0.0, 1.0, 0.0, nan, nan],
                [0.0, 1.0, 0.0, 2.0, 0.0, 3.0, nan, nan],
                [2.0, 0.0, 3.0, 0.0, 4.0, 0.0, nan, nan],
                [0.0, 4.0, 0.0, 1.0, 0.0, 2.0, nan, nan]
            ]))

        target_gm = GridMapping.regular((8, 6), (49.8, 9.9), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, 2.0, 0.0, 3.0, 0.0, 4.0, 0.0],
                [nan, nan, 0.0, 4.0, 0.0, 1.0, 0.0, 2.0],
                [nan, nan, 1.0, 0.0, 2.0, 0.0, 3.0, 0.0],
                [nan, nan, 0.0, 3.0, 0.0, 4.0, 0.0, 1.0],
                [nan, nan, 4.0, 0.0, 1.0, 0.0, 2.0, 0.0],
                [nan, nan, nan, nan, nan, nan, nan, nan]
            ]))


# test/core/resampling/test_rectify.py

import unittest
from typing import Tuple

import numpy as np
# noinspection PyUnresolvedReferences
import xarray as xr

from test.sampledata import SourceDatasetMixin
from test.sampledata import create_s2plus_dataset
from xcube.core.gridmapping import CRS_WGS84
from xcube.core.gridmapping import GridMapping
from xcube.core.resampling import rectify_dataset

nan = np.nan


# noinspection PyMethodMayBeStatic
class RectifyDatasetTest(SourceDatasetMixin, unittest.TestCase):

    def test_rectify_2x2_to_default(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(4, 4),
                                        xy_min=(-1, 49),
                                        xy_res=2,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        # target_ds = rectify_dataset(source_ds)

        rad = target_ds.rad
        # lon, lat, rad = self._assert_shape_and_dim(target_ds, (4, 4))
        # np.testing.assert_almost_equal(lon.values,
        #                                np.array([0., 2., 4., 6.],
        #                                         dtype=lon.dtype))
        # np.testing.assert_almost_equal(lat.values,
        #                                np.array([56., 54., 52., 50.],
        #                                         dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values,
                                       np.array([
                                           [nan, nan, nan, nan],
                                           [nan, 1.0, 2.0, nan],
                                           [3.0, 3.0, 2.0, nan],
                                           [nan, 4.0, nan, nan]
                                       ],
                                           dtype=rad.dtype))

    def test_rectify_2x2_to_7x7(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(7, 7),
                                        xy_min=(-0.5, 49.5),
                                        xy_res=1.0,
                                        crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (7, 7))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5., 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55., 54., 53., 52., 51., 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values,
                                       np.array([
                                           [nan, 1.0, nan, nan, nan, nan, nan],
                                           [nan, 1.0, 1.0, nan, nan, nan, nan],
                                           [nan, 1.0, 1.0, 1.0, 2.0, nan, nan],
                                           [nan, 3.0, 3.0, 1.0, 2.0, 2.0, 2.0],
                                           [3.0, 3.0, 3.0, 4.0, 2.0, nan, nan],
                                           [nan, 3.0, 4.0, 4.0, nan, nan, nan],
                                           [nan, nan, 4.0, nan, nan, nan, nan],
                                       ], dtype=rad.dtype))

    def test_rectify_2x2_to_7x7_subset(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(7, 7),
                                        xy_min=(1.5, 50.5),
                                        xy_res=1.0,
                                        crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        lon, lat, rad = self._assert_shape_and_dim(target_ds, (7, 7))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([2.0, 3.0, 4.0, 5., 6., 7., 8.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([57., 56., 55., 54., 53., 52., 51.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values,
                                       np.array([
                                           [nan, nan, nan, nan, nan, nan, nan],
                                           [nan, nan, nan, nan, nan, nan, nan],
                                           [1.0, nan, nan, nan, nan, nan, nan],
                                           [1.0, 1.0, 2.0, nan, nan, nan, nan],
                                           [3.0, 1.0, 2.0, 2.0, 2.0, nan, nan],
                                           [3.0, 4.0, 2.0, nan, nan, nan, nan],
                                           [4.0, 4.0, nan, nan, nan, nan, nan],
                                       ], dtype=rad.dtype))

    def test_rectify_2x2_to_13x13(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_j_axis_up(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        is_j_axis_up=True)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([50., 50.5, 51., 51.5, 52., 52.5, 53., 53.5, 54., 54.5, 55.,
                                                 55.5, 56.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype)[::-1])

    def test_rectify_2x2_to_13x13_j_axis_up_dask_5x5(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=5,
                                        is_j_axis_up=True)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((5, 5, 3), (5, 5, 3)))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([50., 50.5, 51., 51.5, 52., 52.5, 53., 53.5, 54., 54.5, 55.,
                                                 55.5, 56.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype)[::-1])

    def test_rectify_2x2_to_13x13_dask_7x7(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=7)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((7, 6), (7, 6)))

        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_dask_5x5(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=5)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((5, 5, 3), (5, 5, 3)))

        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_dask_3x13(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=(3, 13))

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((13,), (3, 3, 3, 3, 1)))

        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_dask_13x3(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=(13, 3))

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((3, 3, 3, 3, 1), (13,)))

        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_antimeridian(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords_antimeridian()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(177.75, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)

        self.assertEqual(True, target_gm.is_lon_360)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        self.assertIsNotNone(target_ds)
        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([178., 178.5, 179., 179.5, 180., -179.5, -179., -178.5,
                                                 -178., -177.5, -177., -176.5, -176.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_output_ij_names(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm, output_ij_names=('source_i', 'source_j'))

        lon, lat, rad, source_i, source_j = self._assert_shape_and_dim(target_ds, (13, 13),
                                                                       var_names=('rad', 'source_i', 'source_j'))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))
        np.testing.assert_almost_equal(np.floor(source_i.values + 0.5), self.expected_i_13x13())
        np.testing.assert_almost_equal(np.floor(source_j.values + 0.5), self.expected_j_13x13())

    def test_rectify_2x2_to_13x13_output_ij_names_dask(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=5)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm, output_ij_names=('source_i', 'source_j'))
        lon, lat, rad, source_i, source_j = self._assert_shape_and_dim(target_ds, (13, 13),
                                                                       chunks=((5, 5, 3), (5, 5, 3)),
                                                                       var_names=('rad', 'source_i', 'source_j'))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        # print('source_i:', np.floor(source_i.values + 0.5))
        # print('source_j:', np.floor(source_j.values + 0.5))
        np.testing.assert_almost_equal(np.floor(source_i.values + 0.5), self.expected_i_13x13())
        np.testing.assert_almost_equal(np.floor(source_j.values + 0.5), self.expected_j_13x13())
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_none(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(10.0, 50.0),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        self.assertIsNone(target_ds)

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-10.0, 50.0),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        self.assertIsNone(target_ds)

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(0.0, 58.0),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        self.assertIsNone(target_ds)

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(0.0, 42.0),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        self.assertIsNone(target_ds)

    def _assert_shape_and_dim(self, target_ds, size, chunks=None, var_names=('rad',)) -> Tuple[xr.DataArray, ...]:
        w, h = size

        self.assertIn('lon', target_ds)
        lon = target_ds['lon']
        self.assertEqual((w,), lon.shape)
        self.assertEqual(('lon',), lon.dims)

        self.assertIn('lat', target_ds)
        lat = target_ds['lat']
        self.assertEqual((h,), lat.shape)
        self.assertEqual(('lat',), lat.dims)

        # noinspection PyShadowingBuiltins
        vars = []
        for var_name in var_names:
            self.assertIn(var_name, target_ds)
            var = target_ds[var_name]
            self.assertEqual((h, w), var.shape)
            self.assertEqual(('lat', 'lon'), var.dims)
            self.assertEqual(chunks, var.chunks)
            vars.append(var)

        return (lon, lat, *vars)

    def test_compute_and_extract_source_pixels(self):
        from xcube.core.resampling.rectify import _compute_ij_images_numpy_parallel
        from xcube.core.resampling.rectify import _compute_var_image_numpy_parallel
        self._assert_compute_and_extract_source_pixels(_compute_ij_images_numpy_parallel,
                                                       _compute_var_image_numpy_parallel, False)
        from xcube.core.resampling.rectify import _compute_ij_images_numpy_sequential
        from xcube.core.resampling.rectify import _compute_var_image_numpy_sequential
        self._assert_compute_and_extract_source_pixels(_compute_ij_images_numpy_sequential,
                                                       _compute_var_image_numpy_sequential, False)

    def test_compute_and_extract_source_pixels_j_axis_up(self):
        from xcube.core.resampling.rectify import _compute_ij_images_numpy_parallel
        from xcube.core.resampling.rectify import _compute_var_image_numpy_parallel
        self._assert_compute_and_extract_source_pixels(_compute_ij_images_numpy_parallel,
                                                       _compute_var_image_numpy_parallel, True)
        from xcube.core.resampling.rectify import _compute_ij_images_numpy_sequential
        from xcube.core.resampling.rectify import _compute_var_image_numpy_sequential
        self._assert_compute_and_extract_source_pixels(_compute_ij_images_numpy_sequential,
                                                       _compute_var_image_numpy_sequential, True)

    def _assert_compute_and_extract_source_pixels(self,
                                                  compute_ij_images,
                                                  compute_var_image,
                                                  is_j_axis_up: bool):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        dst_src_ij = np.full((2, 13, 13), np.nan, dtype=np.float64)
        dst_x_offset = -0.25
        dst_y_offset = 49.75 if is_j_axis_up else 56.25
        dst_x_scale = 0.5
        dst_y_scale = 0.5 if is_j_axis_up else -0.5
        compute_ij_images(source_ds.lon.values,
                          source_ds.lat.values,
                          0,
                          0,
                          dst_src_ij,
                          dst_x_offset,
                          dst_y_offset,
                          dst_x_scale,
                          dst_y_scale,
                          1e-5)

        if not is_j_axis_up:
            np.testing.assert_almost_equal(np.floor(dst_src_ij[0] + 0.5), self.expected_i_13x13())
            np.testing.assert_almost_equal(np.floor(dst_src_ij[1] + 0.5), self.expected_j_13x13())
        else:
            np.testing.assert_almost_equal(np.floor(dst_src_ij[0] + 0.5), self.expected_i_13x13()[::-1])
            np.testing.assert_almost_equal(np.floor(dst_src_ij[1] + 0.5), self.expected_j_13x13()[::-1])

        target_rad = np.full((13, 13), np.nan, dtype=np.float64)

        compute_var_image(source_ds.rad.values,
                          dst_src_ij,
                          target_rad)

        if not is_j_axis_up:
            np.testing.assert_almost_equal(target_rad, self.expected_rad_13x13(target_rad.dtype))
        else:
            np.testing.assert_almost_equal(target_rad, self.expected_rad_13x13(target_rad.dtype)[::-1])

    def expected_i_13x13(self):
        return np.array([
            [nan, nan, 0.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, 0.0, 0.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, nan, nan, nan, nan, nan],
            [nan, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, nan, nan, nan, nan],
            [nan, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, nan, nan],
            [nan, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
            [nan, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan],
            [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan],
            [nan, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, nan, 1.0, nan, nan, nan, nan, nan, nan, nan, nan],
        ],
            dtype=np.float64)

    def expected_j_13x13(self):
        return np.array([
            [nan, nan, 0.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, 0.0, 0.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, nan, nan, nan, nan, nan],
            [nan, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, nan, nan, nan, nan],
            [nan, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, nan, nan],
            [nan, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
            [nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, nan, nan],
            [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, nan, nan, nan],
            [nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, nan, 1.0, nan, nan, nan, nan, nan, nan, nan, nan],
        ],
            dtype=np.float64)

    def expected_rad_13x13(self, dtype):
        return np.array([
            [nan, nan, 1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan],
            [nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, nan, nan, nan, nan],
            [nan, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, nan, nan],
            [nan, 3.0, 3.0, 3.0, 3.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0],
            [nan, 3.0, 3.0, 3.0, 3.0, 3.0, 1.0, 2.0, 2.0, 2.0, 2.0, nan, nan],
            [3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 2.0, 2.0, 2.0, nan, nan, nan],
            [nan, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 2.0, nan, nan, nan, nan],
            [nan, nan, 3.0, 4.0, 4.0, 4.0, 4.0, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, 4.0, 4.0, 4.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, nan, 4.0, nan, nan, nan, nan, nan, nan, nan, nan],
        ], dtype=dtype)


class RectifySentinel2DatasetTest(SourceDatasetMixin, unittest.TestCase):

    def test_rectify_dataset(self):
        source_ds = create_s2plus_dataset()

        expected_data = np.array([
            [nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, 0.019001, 0.019001, 0.008999, 0.012001, 0.012001, 0.022999, nan, nan],
            [nan, 0.021, 0.021, 0.009998, 0.009998, 0.008999, 0.022999, nan, nan],
            [nan, 0.022999, 0.022999, 0.007999, 0.007999, 0.008999, 0.023998, nan, nan],
            [nan, 0.022999, 0.022999, 0.007, 0.007, 0.009998, 0.021, nan, nan],
            [nan, nan, nan, nan, nan, nan, nan, nan, nan]
        ])

        source_gm = GridMapping.from_dataset(source_ds, prefer_crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds,
                                    source_gm=source_gm,
                                    tile_size=None)
        self.assertEqual(None, target_ds.rrs_665.chunks)
        print(repr(target_ds.rrs_665.values))
        np.testing.assert_almost_equal(target_ds.rrs_665.values,
                                       expected_data, decimal=3)

        target_ds = rectify_dataset(source_ds,
                                    source_gm=source_gm,
                                    tile_size=5)
        self.assertEqual(((5, 1), (5, 4)), target_ds.rrs_665.chunks)
        np.testing.assert_almost_equal(target_ds.rrs_665.values,
                                       expected_data, decimal=3)

        target_ds = rectify_dataset(source_ds,
                                    source_gm=source_gm,
                                    tile_size=None,
                                    is_j_axis_up=True)
        self.assertEqual(None, target_ds.rrs_665.chunks)
        np.testing.assert_almost_equal(target_ds.rrs_665.values,
                                       expected_data[::-1], decimal=3)

        target_ds = rectify_dataset(source_ds,
                                    source_gm=source_gm,
                                    tile_size=5,
                                    is_j_axis_up=True)
        self.assertEqual(((5, 1), (5, 4)), target_ds.rrs_665.chunks)
        np.testing.assert_almost_equal(target_ds.rrs_665.values,
                                       expected_data[::-1], decimal=3)


# test/core/resampling/test_spatial.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.gridmapping import CRS_WGS84
from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.resampling import resample_in_space

nan = np.nan


class ResampleInSpaceTest(unittest.TestCase):
    # TODO: write more tests

    # noinspection PyMethodMayBeStatic
    def test_affine_transform_dataset(self):
        source_ds = new_cube()
        source_gm = GridMapping.from_dataset(source_ds)
        target_gm = GridMapping.regular(size=(8, 4),
                                        xy_min=(0, 0),
                                        xy_res=2,
                                        crs=CRS_WGS84)

        # TODO: assert stuff
        resample_in_space(source_ds, source_gm, target_gm)


# test/core/resampling/test_temporal.py

import unittest

import numpy as np
import pandas as pd

from test.sampledata import new_test_dataset
from xcube.core.chunk import chunk_dataset
from xcube.core.resampling import resample_in_time
from xcube.core.schema import CubeSchema


class ResampleInTimeTest(unittest.TestCase):

    def setUp(self) -> None:
        num_times = 30

        time = []
        periods = ['1D', '1D', '3D', '4D', '2D']
        t = pd.to_datetime('2017-07-01T10:30:15Z', utc=True)
        for i in range(num_times):
            time.append(t.isoformat())
            t += pd.to_timedelta(periods[i % len(periods)])

        temperature, precipitation = zip(*[(272 + 0.1 * i, 120 - 0.2 * i) for i in range(num_times)])

        input_cube = new_test_dataset(time, temperature=temperature, precipitation=precipitation)
        input_cube = chunk_dataset(input_cube, chunk_sizes=dict(time=1, lat=90, lon=180))
        self.input_cube = input_cube

    def test_resample_in_time_min_max(self):
        resampled_cube = resample_in_time(self.input_cube, '2W', ['min', 'max'])
        self.assertIsNot(resampled_cube, self.input_cube)
        self.assertIn('time', resampled_cube)
        self.assertIn('temperature_min', resampled_cube)
        self.assertIn('temperature_max', resampled_cube)
        self.assertIn('precipitation_min', resampled_cube)
        self.assertIn('precipitation_max', resampled_cube)
        self.assertEqual(('time',), resampled_cube.time.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_min.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_max.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_min.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_max.dims)
        self.assertEqual((6,), resampled_cube.time.shape)
        self.assertEqual((6, 180, 360), resampled_cube.temperature_min.shape)
        self.assertEqual((6, 180, 360), resampled_cube.temperature_max.shape)
        self.assertEqual((6, 180, 360), resampled_cube.precipitation_min.shape)
        self.assertEqual((6, 180, 360), resampled_cube.precipitation_max.shape)
        np.testing.assert_equal(resampled_cube.time.values,
                                np.array(
                                    ['2017-06-25T00:00:00', '2017-07-09T00:00:00',
                                     '2017-07-23T00:00:00', '2017-08-06T00:00:00',
                                     '2017-08-20T00:00:00', '2017-09-03T00:00:00'], dtype=np.datetime64))
        np.testing.assert_allclose(resampled_cube.temperature_min.values[..., 0, 0],
                                   np.array([272.0, 272.4, 273.0, 273.8, 274.4, 274.9]))
        np.testing.assert_allclose(resampled_cube.temperature_max.values[..., 0, 0],
                                   np.array([272.3, 272.9, 273.7, 274.3, 274.8, 274.9]))
        np.testing.assert_allclose(resampled_cube.precipitation_min.values[..., 0, 0],
                                   np.array([119.4, 118.2, 116.6, 115.4, 114.4, 114.2]))
        np.testing.assert_allclose(resampled_cube.precipitation_max.values[..., 0, 0],
                                   np.array([120.0, 119.2, 118.0, 116.4, 115.2, 114.2]))

        schema = CubeSchema.new(resampled_cube)
        self.assertEqual(3, schema.ndim)
        self.assertEqual(('time', 'lat', 'lon'), schema.dims)
        self.assertEqual((6, 180, 360), schema.shape)
        self.assertEqual((1, 90, 180), schema.chunks)

    def test_resample_in_time_p90_dask(self):
        # "percentile_<p>" can currently only be used with numpy rather than chunked dask arrays:

        # TypeError raised on Windows, ValueError on Linux
        with self.assertRaises(Exception):
            # noinspection PyUnusedLocal
            resampled_cube = resample_in_time(self.input_cube, '2W', 'percentile_90')

    # TODO (forman): the call to resample_in_time() takes forever,
    #                this is not xcube, but may be an issue in dask 0.14 or dask 2.8.
    # def test_resample_in_time_p90_numpy(self):
    #     # "percentile_<p>" can currently only be used with numpy, so compute() first:
    #     input_cube = self.input_cube.compute()
    #     resampled_cube = resample_in_time(input_cube, '2W', 'percentile_90')
    #     self.assertIsNot(resampled_cube, self.input_cube)
    #     self.assertIn('time', resampled_cube)
    #     self.assertIn('temperature_p90', resampled_cube)
    #     self.assertIn('precipitation_p90', resampled_cube)
    #     self.assertEqual(('time',), resampled_cube.time.dims)
    #     self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_p90.dims)
    #     self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_p90.dims)
    #     self.assertEqual((6,), resampled_cube.time.shape)
    #     self.assertEqual((6, 180, 360), resampled_cube.temperature_p90.shape)
    #     self.assertEqual((6, 180, 360), resampled_cube.precipitation_p90.shape)
    #     np.testing.assert_equal(resampled_cube.time.values,
    #                             np.array(
    #                                 ['2017-06-25T00:00:00Z', '2017-07-09T00:00:00Z',
    #                                  '2017-07-23T00:00:00Z', '2017-08-06T00:00:00Z',
    #                                  '2017-08-20T00:00:00Z', '2017-09-03T00:00:00Z'], dtype=np.datetime64))
    #     np.testing.assert_allclose(resampled_cube.temperature_p90.values[..., 0, 0],
    #                                np.array([272.3, 272.9, 273.7, 274.3, 274.8, 274.9]))
    #     np.testing.assert_allclose(resampled_cube.precipitation_p90.values[..., 0, 0],
    #                                np.array([120.0, 119.2, 118.0, 116.4, 115.2, 114.2]))
    #
    #     schema = CubeSchema.new(resampled_cube)
    #     self.assertEqual(3, schema.ndim)
    #     self.assertEqual(('time', 'lat', 'lon'), schema.dims)
    #     self.assertEqual((6, 180, 360), schema.shape)
    #     self.assertEqual((1, 90, 180), schema.chunks)

    def test_resample_in_time_with_time_chunk_size(self):
        resampled_cube = resample_in_time(self.input_cube, '2D', ['min', 'max'], time_chunk_size=5)
        schema = CubeSchema.new(resampled_cube)
        self.assertEqual(3, schema.ndim)
        self.assertEqual(('time', 'lat', 'lon'), schema.dims)
        self.assertEqual((33, 180, 360), schema.shape)
        self.assertEqual((5, 90, 180), schema.chunks)

    def test_resample_f_all(self):
        resampled_cube = resample_in_time(self.input_cube, 'all', ['min', 'max'])
        self.assertIsNot(resampled_cube, self.input_cube)
        self.assertIn('time', resampled_cube)
        self.assertIn('temperature_min', resampled_cube)
        self.assertIn('temperature_max', resampled_cube)
        self.assertIn('precipitation_min', resampled_cube)
        self.assertIn('precipitation_max', resampled_cube)
        self.assertEqual(('time',), resampled_cube.time.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_min.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_max.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_min.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_max.dims)
        self.assertEqual((1,), resampled_cube.time.shape)
        self.assertEqual((1, 180, 360), resampled_cube.temperature_min.shape)
        self.assertEqual((1, 180, 360), resampled_cube.temperature_max.shape)
        self.assertEqual((1, 180, 360), resampled_cube.precipitation_min.shape)
        self.assertEqual((1, 180, 360), resampled_cube.precipitation_max.shape)
        np.testing.assert_allclose(resampled_cube.temperature_min.values[..., 0, 0],
                                   np.array([272.0]))
        np.testing.assert_allclose(resampled_cube.temperature_max.values[..., 0, 0],
                                   np.array([274.9]))
        np.testing.assert_allclose(resampled_cube.precipitation_min.values[..., 0, 0],
                                   np.array([114.2]))
        np.testing.assert_allclose(resampled_cube.precipitation_max.values[..., 0, 0],
                                   np.array([120.0]))

        schema = CubeSchema.new(resampled_cube)
        self.assertEqual(3, schema.ndim)
        self.assertEqual(('time', 'lat', 'lon'), schema.dims)
        self.assertEqual((1, 180, 360), schema.shape)


# test/core/store/test_datatype.py

import copy
import unittest

import geopandas
import xarray

from xcube.core.mldataset import MultiLevelDataset
from xcube.core.store.datatype import ANY_TYPE, DataTypeLike
from xcube.core.store.datatype import DATASET_TYPE
from xcube.core.store.datatype import DataType
from xcube.core.store.datatype import GEO_DATA_FRAME_TYPE
from xcube.core.store.datatype import MULTI_LEVEL_DATASET_TYPE
from xcube.util.jsonschema import JsonStringSchema


class A:
    pass


class B(A):
    pass


class C:
    pass


class DataTypeTest(unittest.TestCase):
    def test_normalize_to_any(self):
        self.assertNormalizeOk('any',
                               ANY_TYPE,
                               object,
                               'any',
                               ('any', '*', 'object', 'builtins.object'))
        self.assertNormalizeOk(None,
                               ANY_TYPE,
                               object,
                               'any',
                               ('any', '*', 'object', 'builtins.object'))
        self.assertNormalizeOk(type(None),
                               ANY_TYPE,
                               object,
                               'any',
                               ('any', '*', 'object', 'builtins.object'))

    def test_normalize_to_dataset(self):
        self.assertNormalizeOk('dataset',
                               DATASET_TYPE,
                               xarray.Dataset,
                               'dataset',
                               ('dataset',
                                'xarray.Dataset',
                                'xarray.core.dataset.Dataset'))

    def test_normalize_to_mldataset(self):
        self.assertNormalizeOk('mldataset',
                               MULTI_LEVEL_DATASET_TYPE,
                               MultiLevelDataset,
                               'mldataset',
                               ('mldataset',
                                'xcube.MultiLevelDataset',
                                'xcube.core.mldataset.MultiLevelDataset'))

    def test_normalize_to_geodataframe(self):
        self.assertNormalizeOk('geodataframe',
                               GEO_DATA_FRAME_TYPE,
                               geopandas.GeoDataFrame,
                               'geodataframe',
                               ('geodataframe',
                                'geopandas.GeoDataFrame',
                                'geopandas.geodataframe.GeoDataFrame'))

    def assertNormalizeOk(self,
                          data_type: DataTypeLike,
                          expected_data_type,
                          expected_dtype,
                          expected_alias,
                          expected_aliases):
        data_type = DataType.normalize(data_type)
        self.assertIs(expected_data_type, data_type)
        self.assertIs(expected_dtype, data_type.dtype)
        self.assertEqual(expected_alias, data_type.alias)
        self.assertEqual(expected_alias, str(data_type))
        self.assertEqual(f'{expected_alias!r}', repr(data_type))
        self.assertEqual(expected_aliases, data_type.aliases)
        for other_alias in data_type.aliases:
            self.assertIs(data_type,
                          DataType.normalize(other_alias))
        self.assertIs(expected_data_type,
                      DataType.normalize(expected_dtype))
        self.assertIs(expected_data_type,
                      DataType.normalize(expected_data_type))

    def test_normalize_non_default_types(self):
        data_type = DataType.normalize(str)
        self.assertIs(str, data_type.dtype)
        self.assertEqual('builtins.str', data_type.alias)

    def test_normalize_failure(self):
        with self.assertRaises(ValueError) as cm:
            DataType.normalize('Gnartz')
        self.assertEqual("unknown data type 'Gnartz'",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            DataType.normalize(42)
        self.assertEqual("cannot convert 42 into a data type",
                         f'{cm.exception}')

    def test_equality(self):
        self.assertIs(DATASET_TYPE, DATASET_TYPE)
        self.assertEqual(DATASET_TYPE, DATASET_TYPE)

        self.assertEqual(copy.deepcopy(DATASET_TYPE), DATASET_TYPE)
        self.assertNotEqual(MULTI_LEVEL_DATASET_TYPE, DATASET_TYPE)

        self.assertEqual(hash(copy.deepcopy(DATASET_TYPE)), hash(DATASET_TYPE))
        self.assertNotEqual(hash(MULTI_LEVEL_DATASET_TYPE), hash(DATASET_TYPE))

    def test_is_sub_type_of(self):
        self.assertTrue(ANY_TYPE.is_sub_type_of(ANY_TYPE))
        self.assertFalse(ANY_TYPE.is_sub_type_of(DATASET_TYPE))
        self.assertFalse(ANY_TYPE.is_sub_type_of(MULTI_LEVEL_DATASET_TYPE))

        self.assertTrue(DATASET_TYPE.is_sub_type_of(ANY_TYPE))
        self.assertTrue(DATASET_TYPE.is_sub_type_of(DATASET_TYPE))
        self.assertFalse(DATASET_TYPE.is_sub_type_of(MULTI_LEVEL_DATASET_TYPE))

        a_type = DataType.normalize(A)
        b_type = DataType.normalize(B)
        c_type = DataType.normalize(C)
        self.assertTrue(a_type.is_sub_type_of(a_type))
        self.assertFalse(a_type.is_sub_type_of(b_type))
        self.assertFalse(a_type.is_sub_type_of(c_type))
        self.assertTrue(b_type.is_sub_type_of(a_type))
        self.assertTrue(b_type.is_sub_type_of(b_type))
        self.assertFalse(b_type.is_sub_type_of(c_type))
        self.assertFalse(c_type.is_sub_type_of(a_type))
        self.assertFalse(c_type.is_sub_type_of(b_type))
        self.assertTrue(c_type.is_sub_type_of(c_type))

    def test_is_super_type_of(self):
        self.assertTrue(ANY_TYPE.is_super_type_of(ANY_TYPE))
        self.assertTrue(ANY_TYPE.is_super_type_of(DATASET_TYPE))
        self.assertTrue(ANY_TYPE.is_super_type_of(DATASET_TYPE))

        self.assertTrue(ANY_TYPE.is_super_type_of(DATASET_TYPE))
        self.assertTrue(DATASET_TYPE.is_super_type_of(DATASET_TYPE))
        self.assertFalse(MULTI_LEVEL_DATASET_TYPE.is_super_type_of(DATASET_TYPE))

        a_type = DataType.normalize(A)
        b_type = DataType.normalize(B)
        c_type = DataType.normalize(C)
        self.assertTrue(a_type.is_super_type_of(a_type))
        self.assertTrue(a_type.is_super_type_of(b_type))
        self.assertFalse(a_type.is_super_type_of(c_type))
        self.assertFalse(b_type.is_super_type_of(a_type))
        self.assertTrue(b_type.is_super_type_of(b_type))
        self.assertFalse(b_type.is_super_type_of(c_type))
        self.assertFalse(c_type.is_super_type_of(a_type))
        self.assertFalse(c_type.is_super_type_of(b_type))
        self.assertTrue(c_type.is_super_type_of(c_type))

    def test_schema(self):
        self.assertIsInstance(DATASET_TYPE.get_schema(), JsonStringSchema)


# test/core/store/test_descriptor.py

import unittest
from typing import Type

import jsonschema
import numpy as np

from xcube.core.mldataset import BaseMultiLevelDataset
from xcube.core.new import new_cube
from xcube.core.store.datatype import DataType
from xcube.core.store.descriptor import DataDescriptor, _attrs_to_json, MultiLevelDatasetDescriptor
from xcube.core.store.descriptor import DatasetDescriptor
from xcube.core.store.descriptor import GeoDataFrameDescriptor
from xcube.core.store.descriptor import VariableDescriptor
from xcube.core.store.descriptor import new_data_descriptor
from xcube.util.jsonschema import JsonBooleanSchema
from xcube.util.jsonschema import JsonObjectSchema


class NewDataDescriptorTest(unittest.TestCase):

    def test_new_dataset_descriptor(self):
        cube = new_cube(variables=dict(a=4.1, b=7.4))
        descriptor = new_data_descriptor('cube', cube)
        self.assertExpectedDescriptor(descriptor,
                                      DatasetDescriptor,
                                      'dataset')

    def test_new_ml_dataset_descriptor(self):
        cube = new_cube(variables=dict(a=4.1, b=7.4))
        ml_cube = BaseMultiLevelDataset(cube)
        descriptor = new_data_descriptor('cube', ml_cube)
        self.assertExpectedDescriptor(descriptor,
                                      MultiLevelDatasetDescriptor,
                                      'mldataset')

    def assertExpectedDescriptor(self,
                                 descriptor: DataDescriptor,
                                 expected_type: Type[DataDescriptor],
                                 expected_data_type_alias: str):
        self.assertIsInstance(descriptor, DatasetDescriptor)
        self.assertIsInstance(descriptor, expected_type)
        self.assertIsInstance(descriptor.data_type, DataType)
        self.assertEqual(expected_data_type_alias, descriptor.data_type.alias)
        self.assertEqual('cube', descriptor.data_id)
        self.assertEqual((-180.0, -90.0, 180.0, 90.0), descriptor.bbox)
        self.assertIsNone(descriptor.open_params_schema)
        self.assertEqual(('2010-01-01', '2010-01-06'), descriptor.time_range)
        self.assertEqual('1D', descriptor.time_period)
        self.assertEqual(1.0, descriptor.spatial_res)
        self.assertIsNotNone(descriptor.coords)
        self.assertEqual({'time': 5, 'lat': 180, 'lon': 360, 'bnds': 2}, descriptor.dims)
        self.assertIsNotNone(descriptor.data_vars)


class DataDescriptorTest(unittest.TestCase):

    def test_from_dict_no_data_id(self):
        with self.assertRaises(jsonschema.exceptions.ValidationError):
            descriptor_dict = dict()
            DataDescriptor.from_dict(descriptor_dict)

    def test_from_dict_no_data_type(self):
        with self.assertRaises(jsonschema.exceptions.ValidationError):
            descriptor_dict = dict(data_id='id')
            DataDescriptor.from_dict(descriptor_dict)

    def test_from_dict_unknown_data_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='tsr')
        with self.assertRaises(ValueError) as cm:
            DataDescriptor.from_dict(descriptor_dict)
        self.assertEqual("unknown data type 'tsr'", f'{cm.exception}')

    def test_from_dict_dataset_data_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='dataset')
        descriptor = DataDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertTrue(DatasetDescriptor, type(descriptor))
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)

    def test_from_dict_geodataframe_data_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='geodataframe')
        descriptor = DataDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertTrue(GeoDataFrameDescriptor, type(descriptor))
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('geodataframe', descriptor.data_type.alias)

    def test_from_dict_full(self):
        descriptor_dict = dict(
            data_id='CHL.zarr',
            data_type='dataset',
            crs='EPSG:4326',
            bbox=(10., 20., 30., 40.),
            time_range=('2017-06-05', '2017-06-27'),
            time_period='2D',
            open_params_schema=dict(
                type="object",
                properties=dict(
                    variable_names=dict(
                        type='array',
                        items=dict(type='string')
                    )
                )
            )
        )
        descriptor = DataDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertEqual('CHL.zarr', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)
        self.assertEqual('EPSG:4326', descriptor.crs)
        self.assertEqual((10., 20., 30., 40.), descriptor.bbox)
        self.assertEqual(('2017-06-05', '2017-06-27'), descriptor.time_range)
        self.assertEqual('2D', descriptor.time_period)
        self.assertEqual('object',
                         descriptor.open_params_schema.get('type', None))

    def test_to_dict(self):
        descriptor = DatasetDescriptor(
            data_id='xyz',
            crs='EPSG:9346',
            bbox=(10., 20., 30., 40.),
            spatial_res=20.,
            time_range=('2017-06-05', '2017-06-27'),
            time_period='daily',
            open_params_schema=JsonObjectSchema(
                properties=dict(
                    consolidated=JsonBooleanSchema(),
                ),
                additional_properties=False,
            )
        )
        descriptor_dict = descriptor.to_dict()
        self.assertEqual(
            {
                'data_id': 'xyz',
                'crs': 'EPSG:9346',
                'data_type': 'dataset',
                'bbox': [10.0, 20.0, 30.0, 40.0],
                'spatial_res': 20.0,
                'time_range': ['2017-06-05', '2017-06-27'],
                'time_period': 'daily',
                'open_params_schema': {
                    'type': 'object',
                    'properties': {
                        'consolidated': {
                            'type': 'boolean'
                        }
                    },
                    'additionalProperties': False,
                },
            },
            descriptor_dict)


class DatasetDescriptorTest(unittest.TestCase):

    def test_get_schema(self):
        schema = DatasetDescriptor.get_schema()
        self.assertIsInstance(schema, JsonObjectSchema)

    def test_from_dict_no_data_id(self):
        descriptor_dict = dict()
        with self.assertRaises(jsonschema.exceptions.ValidationError):
            DatasetDescriptor.from_dict(descriptor_dict)

    def test_from_dict_wrong_data_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='tsr')
        with self.assertRaises(ValueError) as cm:
            DatasetDescriptor.from_dict(descriptor_dict)
        self.assertEqual("unknown data type 'tsr'",
                         f'{cm.exception}')

    def test_from_dict_derived_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='dataset')
        descriptor = DatasetDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)

    def test_from_dict_full(self):
        descriptor_dict = dict(
            data_id='xyz',
            data_type='dataset',
            crs='EPSG:9346',
            bbox=(10., 20., 30., 40.),
            spatial_res=20.,
            time_range=('2017-06-05', '2017-06-27'),
            time_period='daily',
            coords=dict(
                rtdt=dict(
                    name='rtdt',
                    dtype='rj',
                    dims=('rtdt',),
                    attrs=dict(
                        ssd=6,
                        zjgrhgu='hgtr'
                    )
                )
            ),
            dims=dict(x=1, y=2, z=3),
            data_vars=dict(
                xf=dict(
                    name='xf',
                    dtype='rj',
                    dims=('dfjhrt', 'sg'),
                    chunks=(2, 3),
                    attrs=dict(
                        ssd=4,
                        zjgrhgu='dgfrf'
                    )
                )
            ),
            attrs=dict(
                dzus=236,
                tgr7h='rt5',
                df='s8fd4w5'
            ),
            open_params_schema=dict(
                type="object",
                properties=dict(
                    variable_names=dict(
                        type='array',
                        items=dict(
                            type='string'
                        )
                    )
                )
            )
        )
        descriptor = DatasetDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)
        self.assertEqual('EPSG:9346', descriptor.crs)
        self.assertEqual((10., 20., 30., 40.), descriptor.bbox)
        self.assertEqual(20., descriptor.spatial_res)
        self.assertEqual(('2017-06-05', '2017-06-27'), descriptor.time_range)
        self.assertEqual('daily', descriptor.time_period)
        self.assertEqual(1, len(descriptor.coords))
        self.assertTrue('rtdt' in descriptor.coords)
        self.assertEqual(dict(x=1, y=2, z=3), descriptor.dims)
        self.assertEqual(1, len(descriptor.data_vars))
        self.assertTrue('xf' in descriptor.data_vars)
        self.assertEqual(236, descriptor.attrs.get('dzus', None))
        self.assertEqual('rt5', descriptor.attrs.get('tgr7h', None))
        self.assertEqual('s8fd4w5', descriptor.attrs.get('df', None))
        self.assertEqual('object', descriptor.open_params_schema.get('type', None))

    def test_from_dict_var_descriptors_as_dict(self):
        descriptor_dict = dict(
            data_id='xyz',
            data_type='dataset',
            crs='EPSG:4326',
            data_vars=dict(
                A=dict(
                    name='A',
                    dtype='float32',
                    dims=('time', 'lat', 'lon'),
                    chunks=(2, 3),
                    attrs=dict(
                        ssd=4,
                        zjgrhgu='dgfrf'
                    )
                )
            )
        )
        descriptor = DatasetDescriptor.from_dict(descriptor_dict)
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)
        self.assertEqual('EPSG:4326', descriptor.crs)
        self.assertEqual(1, len(descriptor.data_vars))
        self.assertTrue('A' in descriptor.data_vars)
        self.assertIsInstance(descriptor.data_vars.get('A'),
                              VariableDescriptor)

    def test_to_dict(self):
        coords = dict(
            rtdt=VariableDescriptor(
                name='rtdt',
                dtype='rj',
                dims=('rtdt',),
                chunks=(2,),
                attrs=dict(
                    ssd=6,
                    zjgrhgu='hgtr'
                )
            )
        )
        var_descriptors = dict(
            xf=VariableDescriptor(
                name='xf',
                dtype='rj',
                dims=('dfjhrt', 'sg'),
                chunks=(2, 3),
                attrs=dict(
                    ssd=4,
                    zjgrhgu='dgfrf'
                )
            )
        )
        descriptor = DatasetDescriptor(
            data_id='xyz',
            crs='EPSG:9346',
            bbox=(10., 20., 30., 40.),
            spatial_res=20.,
            time_range=('2017-06-05', '2017-06-27'),
            time_period='daily',
            coords=coords,
            dims=dict(
                x=1, y=2, z=3
            ),
            data_vars=var_descriptors,
            attrs=dict(
                dzus=236,
                tgr7h='rt5',
                df='s8fd4w5'
            )
        )
        descriptor_dict = descriptor.to_dict()
        self.assertEqual(
            dict(
                data_id='xyz',
                data_type='dataset',
                crs='EPSG:9346',
                bbox=[10., 20., 30., 40.],
                spatial_res=20.,
                time_range=['2017-06-05', '2017-06-27'],
                time_period='daily',
                coords=dict(
                    rtdt=dict(
                        name='rtdt',
                        dtype='rj',
                        dims=['rtdt', ],
                        chunks=[2, ],
                        attrs=dict(
                            ssd=6,
                            zjgrhgu='hgtr'
                        )
                    )
                ),
                dims=dict(x=1, y=2, z=3),
                data_vars=dict(
                    xf=dict(
                        name='xf',
                        dtype='rj',
                        dims=['dfjhrt', 'sg'],
                        chunks=[2, 3],
                        attrs=dict(
                            ssd=4,
                            zjgrhgu='dgfrf'
                        )
                    )
                ),
                attrs=dict(
                    dzus=236,
                    tgr7h='rt5',
                    df='s8fd4w5'
                )
            ),
            descriptor_dict
        )


class VariableDescriptorTest(unittest.TestCase):

    def test_variable_descriptor_basic(self):
        vd1 = VariableDescriptor('gz',
                                 'zughysz',
                                 ['rtdswgt', 'dref', 'zdrs5ge'],
                                 chunks=(3, 321, 4))
        self.assertEqual('gz', vd1.name)
        self.assertEqual('zughysz', vd1.dtype)
        self.assertEqual(('rtdswgt', 'dref', 'zdrs5ge'), vd1.dims)
        self.assertEqual(3, vd1.ndim)
        self.assertEqual((3, 321, 4), vd1.chunks)
        self.assertEqual(None, vd1.attrs)

        vd3 = VariableDescriptor('gz',
                                 'zughysz',
                                 ['rtdswgt', 'dref', 'zdrs5ge'],
                                 chunks=(3, 321, 4),
                                 attrs={'d': 2, 'zjgu': ''})
        self.assertEqual('gz', vd3.name)
        self.assertEqual('zughysz', vd3.dtype)
        self.assertEqual(('rtdswgt', 'dref', 'zdrs5ge'), vd3.dims)
        self.assertEqual(3, vd3.ndim)
        self.assertEqual((3, 321, 4), vd3.chunks)
        self.assertEqual({'d': 2, 'zjgu': ''}, vd3.attrs)

    def test_variable_descriptor_to_dict(self):
        vd = VariableDescriptor('xf',
                                'rj',
                                ['dfjhrt', 'sg'],
                                chunks=(3, 2),
                                attrs={'ssd': 4, 'zjgrhgu': 'dgfrf', 'fill_value': np.NaN})
        expected = {
            'name': 'xf',
            'dtype': 'rj',
            'dims': ['dfjhrt', 'sg'],
            'chunks': [3, 2],
            'attrs': {
                'ssd': 4,
                'zjgrhgu': 'dgfrf',
                'fill_value': None
            }
        }
        self.assertEqual(expected, vd.to_dict())

    def test_variable_descriptor_from_dict(self):
        vd_as_dict = {
            'name': 'xf',
            'dtype': 'rj',
            'dims': ('dfjhrt', 'sg'),
            'chunks': (3, 2),
            'attrs': {
                'ssd': 4,
                'zjgrhgu': 'dgfrf',
                'fill_value': None
            }
        }
        vd = VariableDescriptor.from_dict(vd_as_dict)
        self.assertEqual('xf', vd.name)
        self.assertEqual('rj', vd.dtype)
        self.assertEqual(('dfjhrt', 'sg'), vd.dims)
        self.assertEqual(2, vd.ndim)
        self.assertEqual((3, 2), vd.chunks)
        self.assertEqual({'ssd': 4, 'zjgrhgu': 'dgfrf', 'fill_value': None}, vd.attrs)

        with self.assertRaises(jsonschema.exceptions.ValidationError):
            VariableDescriptor.from_dict({'name': 'dhgfr', 'dtype': 'ghdst'})

        with self.assertRaises(jsonschema.exceptions.ValidationError):
            VariableDescriptor.from_dict({'name': 'dhgfr', 'dims': ['faer', 'bjunda']})

        with self.assertRaises(jsonschema.exceptions.ValidationError):
            VariableDescriptor.from_dict({'dtype': 'ghdst', 'dims': ['faer', 'bjunda']})


class JsonTest(unittest.TestCase):
    def test_attrs_to_json_numpy(self):
        self.assertEqual(
            {
                "num_bands": 17,
                "_FillValue": None,
                "flag_names": "F1 F2 F3 F4 F5",
                "flag_values": [1, 2, 4, 8, 16]
            },
            _attrs_to_json(
                {
                    "num_bands": np.array(17),
                    "_FillValue": np.nan,
                    "flag_names": "F1 F2 F3 F4 F5",
                    "flag_values": np.array([1, 2, 4, 8, 16])
                }
            )
        )

    def test_attrs_to_json_dask(self):
        import dask.array as da
        self.assertEqual(
            {
                "flag_names": "F1 F2 F3 F4 F5",
                "flag_values": [1, 2, 4, 8, 16]
            },
            _attrs_to_json(
                {
                    "flag_names": "F1 F2 F3 F4 F5",
                    "flag_values": da.from_array([1, 2, 4, 8, 16])
                }
            )
        )


# test/core/store/test_storepool.py

import json
import os
import unittest

import jsonschema
import yaml

from xcube.core.store import DataStore
from xcube.core.store import DataStoreConfig
from xcube.core.store import DataStoreError
from xcube.core.store import DataStoreInstance
from xcube.core.store import DataStorePool
from xcube.core.store import get_data_store_instance


class GetDataStoreTest(unittest.TestCase):

    def test_get_data_store_instance_new_inst(self):
        instance = get_data_store_instance('memory')
        self.assertIsInstance(instance, DataStoreInstance)
        self.assertIsInstance(instance.store, DataStore)
        instance2 = get_data_store_instance('memory')
        self.assertIsNot(instance, instance2)
        self.assertIsNot(instance.store, instance2.store)

    def test_get_data_store_instance_from_pool(self):
        pool = DataStorePool({
            'dir': DataStoreConfig('file',
                                   store_params=dict(root='.'))
        })
        instance = get_data_store_instance('@dir', store_pool=pool)
        self.assertTrue(hasattr(instance.store, 'root'))
        # noinspection PyUnresolvedReferences
        self.assertTrue(os.path.isabs(instance.store.root))
        self.assertTrue(os.path.isdir(instance.store.root))
        instance2 = get_data_store_instance('@dir', store_pool=pool)
        self.assertIs(instance, instance2)

    def test_get_data_store_instance_from_pool_with_params(self):
        pool = DataStorePool({
            '@dir': DataStoreConfig('file',
                                    store_params=dict(root='.'))
        })
        with self.assertRaises(ValueError) as cm:
            get_data_store_instance(
                '@dir', store_pool=pool, store_params={'auto_mkdir': True}
            )
        self.assertEqual('store_params cannot be given,'
                         ' with store_id ("@dir") referring'
                         ' to a configured store',
                         f'{cm.exception}')

    def test_get_data_store_instance_from_pool_without_pool(self):
        with self.assertRaises(ValueError) as cm:
            get_data_store_instance('@dir')
        self.assertEqual('store_pool must be given,'
                         ' with store_id ("@dir") referring'
                         ' to a configured store',
                         f'{cm.exception}')

    def test_normalize(self):
        pool = DataStorePool({
            '@dir': DataStoreConfig('directory',
                                    store_params=dict(root='.'))
        })
        file_path = '_test-data-stores-pool.json'
        with open(file_path, 'w') as fp:
            json.dump(pool.to_dict(), fp)
        try:
            pool_1 = DataStorePool.normalize(file_path)
            self.assertIsInstance(pool_1, DataStorePool)
            pool_2 = DataStorePool.normalize(pool_1)
            self.assertIs(pool_2, pool_1)
            pool_3 = DataStorePool.normalize(pool_2.to_dict())
            self.assertIsInstance(pool_3, DataStorePool)
        finally:
            os.remove(file_path)

        with self.assertRaises(TypeError):
            # noinspection PyTypeChecker
            DataStorePool.normalize(42)


class DataStoreConfigTest(unittest.TestCase):

    def test_constructor_and_instance_props(self):
        store_config = DataStoreConfig('file',
                                       store_params={'root': '.'},
                                       title='Local',
                                       description='Local files')
        self.assertEqual('file', store_config.store_id)
        self.assertEqual({'root': '.'}, store_config.store_params)
        self.assertEqual('Local', store_config.title)
        self.assertEqual('Local files', store_config.description)

    def test_constructor_asserts(self):
        with self.assertRaises(ValueError) as cm:
            DataStoreConfig('')
        self.assertEqual('store_id must be given', f'{cm.exception}')

        with self.assertRaises(TypeError) as cm:
            # noinspection PyTypeChecker
            DataStoreConfig('directory', store_params=[1, 'B'])
        self.assertEqual("store_params must be an instance"
                         " of <class 'dict'>, was <class 'list'>",
                         f'{cm.exception}')

    def test_to_dict(self):
        store_config = DataStoreConfig('directory',
                                       store_params={'base_dir': '.'},
                                       title='Local',
                                       description='Local files')
        self.assertEqual({'description': 'Local files',
                          'name': 'Local',
                          'store_id': 'directory',
                          'store_params': {'base_dir': '.'}},
                         store_config.to_dict())

    def test_from_dict(self):
        store_config = DataStoreConfig.from_dict({
            'description': 'Local files',
            'title': 'Local',
            'store_id': 'file',
            'store_params': {'root': '.'}
        })
        self.assertIsInstance(store_config, DataStoreConfig)
        self.assertEqual('file', store_config.store_id)
        self.assertEqual({'root': '.'}, store_config.store_params)
        self.assertEqual('Local', store_config.title)
        self.assertEqual('Local files', store_config.description)

    def test_from_dict_with_valid_cost_params(self):
        store_config = DataStoreConfig.from_dict({
            'description': 'Local files',
            'title': 'Local',
            'store_id': 'file',
            'store_params': {'root': '.'},
            'cost_params': {
                'input_pixels_per_punit': 500,
                'output_pixels_per_punit': 100,
                'input_punits_weight': 1.1,
            }
        })
        self.assertIsInstance(store_config, DataStoreConfig)
        self.assertEqual('file', store_config.store_id)
        self.assertEqual({'root': '.'}, store_config.store_params)
        self.assertEqual('Local', store_config.title)
        self.assertEqual('Local files', store_config.description)

    def test_from_dict_with_invalid_cost_params(self):
        with self.assertRaises(DataStoreError):
            DataStoreConfig.from_dict({'description': 'Local files',
                                       'title': 'Local',
                                       'store_id': 'file',
                                       'store_params': {'root': '.'},
                                       'cost_params': {
                                           # Required:
                                           # 'input_pixels_per_punit': 10,
                                           # 'output_pixels_per_punit': 20,
                                       }})


class DataStorePoolTest(unittest.TestCase):
    def test_default_constr(self):
        pool = DataStorePool()
        self.assertEqual([], pool.store_instance_ids)
        self.assertEqual([], pool.store_configs)

    def test_from_dict_empty(self):
        pool = DataStorePool.from_dict({})
        self.assertIsInstance(pool, DataStorePool)
        self.assertEqual([], pool.store_instance_ids)
        self.assertEqual([], pool.store_configs)

    def test_from_dict_no_store_params(self):
        store_configs = {
            "ram-1": {
                "store_id": "memory"
            }
        }
        pool = DataStorePool.from_dict(store_configs)
        self.assertIsInstance(pool, DataStorePool)
        self.assertEqual(["ram-1"], pool.store_instance_ids)
        self.assertIsInstance(pool.get_store_config('ram-1'), DataStoreConfig)

    def test_from_dict_with_bad_dicts(self):
        store_configs = {
            "dir": {
            }
        }
        with self.assertRaises(jsonschema.exceptions.ValidationError) as cm:
            DataStorePool.from_dict(store_configs)
        self.assertTrue(
            "'store_id' is a required property" in f'{cm.exception}',
            msg=f'{cm.exception}')

        store_configs = {
            "dir": {
                "store_id": 10
            }
        }
        with self.assertRaises(jsonschema.exceptions.ValidationError) as cm:
            DataStorePool.from_dict(store_configs)
        self.assertTrue(
            "Failed validating 'type' in schema" in f'{cm.exception}',
            msg=f'{cm.exception}')

    def test_from_json_file(self):
        self._assert_file_ok('json')

    def test_from_yaml_file(self):
        self._assert_file_ok('yaml')

    def test_from_json_file_env(self):
        self._assert_file_ok('json', use_env_vars=True)
        self._assert_file_ok('json',
                             root_1='/test_1',
                             root_2='/test_2',
                             use_env_vars=True)

    def test_from_yaml_file_env(self):
        self._assert_file_ok('yaml', use_env_vars=True)
        self._assert_file_ok('yaml',
                             root_1='/test_1',
                             root_2='/test_2',
                             use_env_vars=True)

    def _assert_file_ok(self,
                        format_name: str,
                        root_1="/root1",
                        root_2="/root2",
                        use_env_vars=False):
        if use_env_vars:
            store_configs = self._get_test_config(
                root_1='${_TEST_ROOT_1}',
                root_2='${_TEST_ROOT_2}'
            )
            import os
            os.environ['_TEST_ROOT_1'] = root_1
            os.environ['_TEST_ROOT_2'] = root_2
        else:
            store_configs = self._get_test_config(
                root_1=root_1,
                root_2=root_2
            )
        path = 'test-store-configs.' + format_name
        with open(path, 'w') as fp:
            mod = yaml if format_name == 'yaml' else json
            mod.dump(store_configs, fp, indent=2)
        try:
            pool = DataStorePool.from_file(path)
            self.assertIsInstance(pool, DataStorePool)
            self.assertEqual(['ram-1', 'ram-2'], pool.store_instance_ids)
            config_1 = pool.get_store_config('ram-1')
            self.assertIsInstance(config_1, DataStoreConfig)
            self.assertEqual(
                {'store_id': 'memory',
                 'store_params': {'root': root_1}},
                config_1.to_dict())
            config_2 = pool.get_store_config('ram-2')
            self.assertIsInstance(config_2, DataStoreConfig)
            self.assertEqual(
                {'store_id': 'memory',
                 'store_params': {'root': root_2}},
                config_2.to_dict())
        finally:
            import os
            os.remove(path)

    @staticmethod
    def _get_test_config(root_1: str, root_2: str):
        return {
            "ram-1": {
                "store_id": "memory",
                "store_params": {
                    "root": root_1
                },
            },
            "ram-2": {
                "store_id": "memory",
                "store_params": {
                    "root": root_2
                },
            }
        }

    def test_get_store(self):
        store_configs = {
            "dir-1": {
                "store_id": "file",
                "store_params": {
                    "root": "./bibo"
                }
            },
        }
        pool = DataStorePool.from_dict(store_configs)
        store = pool.get_store('dir-1')
        self.assertTrue(hasattr(store, 'root'))
        # noinspection PyUnresolvedReferences
        self.assertTrue(os.path.isabs(store.root))
        self.assertFalse(os.path.exists(store.root))
        # Should stay same instance
        self.assertIs(store, pool.get_store('dir-1'))
        self.assertIs(store, pool.get_store('dir-1'))

    def test_get_store_error(self):
        pool = DataStorePool()
        with self.assertRaises(DataStoreError) as cm:
            pool.get_store('dir-1')
        self.assertEqual('Configured data store instance "dir-1" not found.', f'{cm.exception}')

    def test_to_dict(self):
        self.assertEqual({}, DataStorePool().to_dict())
        self.assertEqual({'ram': {'store_id': 'memory'},
                          'dir': {'store_id': 'file',
                                  'store_params': {'base_dir': 'bibo'}}},
                         DataStorePool({'ram': DataStoreConfig(store_id='memory'),
                                        'dir': DataStoreConfig(store_id='file',
                                                               store_params=dict(base_dir="bibo"))
                                        }).to_dict())

    def test_close_all_stores(self):
        store_configs = {
            "ram-1": {
                "store_id": "memory",
            },
        }
        pool = DataStorePool.from_dict(store_configs)
        # Smoke test, we do not expect any visible state changes after close_all_stores()
        pool.close_all_stores()

    def test_add_remove_store_config(self):
        pool = DataStorePool()
        self.assertEqual([], pool.store_instance_ids)
        pool.add_store_config('mem-1', DataStoreConfig('memory'))
        self.assertEqual(['mem-1'], pool.store_instance_ids)
        pool.add_store_config('mem-2', DataStoreConfig('memory'))
        self.assertEqual(['mem-1', 'mem-2'], pool.store_instance_ids)
        pool.add_store_config('mem-1', DataStoreConfig('memory'))
        self.assertEqual(['mem-1', 'mem-2'], pool.store_instance_ids)
        pool.remove_store_config('mem-1')
        self.assertEqual(['mem-2'], pool.store_instance_ids)
        pool.remove_store_config('mem-2')
        self.assertEqual([], pool.store_instance_ids)

    def test_multi_stores_with_params(self):
        """Just test many stores at once"""
        store_configs = {
            "ram-1": {
                "store_id": "memory",
            },
            "ram-2": {
                "store_id": "memory",
            },
            "local-1": {
                "store_id": "file",
                "store_params": {
                    "root": "/home/bibo/datacubes-1",
                }
            },
            "local-2": {
                "store_id": "file",
                "store_params": {
                    "root": "/home/bibo/datacubes-2",
                }
            },
        }
        pool = DataStorePool.from_dict(store_configs)
        self.assertIsInstance(pool, DataStorePool)
        self.assertEqual(["local-1", "local-2", "ram-1", "ram-2"], pool.store_instance_ids)
        for instance_id in pool.store_instance_ids:
            self.assertTrue(pool.has_store_instance(instance_id))
            self.assertIsInstance(pool.get_store_config(instance_id), DataStoreConfig)
            self.assertIsInstance(pool.get_store(instance_id), DataStore)

    def test_get_store_instance_id(self):
        store_params_1 = {
            "root": "./bibo"
        }
        ds_config_1 = DataStoreConfig(store_id='file',
                                      store_params=store_params_1)
        ds_configs = {'dir-1': ds_config_1}
        pool = DataStorePool(ds_configs)

        store_params_2 = {
            "root": "./babo"
        }
        ds_config_2 = DataStoreConfig(store_id='file',
                                      store_params=store_params_2)
        ds_config_3 = DataStoreConfig(store_id='file',
                                      store_params=store_params_1,
                                      title='A third configuration')

        self.assertEqual('dir-1', pool.get_store_instance_id(ds_config_1))
        self.assertEqual('dir-1', pool.get_store_instance_id(ds_config_1,
                                                             strict_check=True))

        self.assertIsNone(pool.get_store_instance_id(ds_config_2))

        self.assertEqual('dir-1', pool.get_store_instance_id(ds_config_3))
        self.assertIsNone(pool.get_store_instance_id(ds_config_3,
                                                     strict_check=True))


# xcube/__init__.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from .version import version

__version__ = version


# xcube/constants.py

# The MIT License (MIT)
# Copyright (c) 2019 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import logging
import math

PLUGIN_ENTRY_POINT_GROUP_NAME = 'xcube_plugins'
PLUGIN_MODULE_PREFIX = 'xcube_'
PLUGIN_MODULE_NAME = 'plugin'
PLUGIN_MODULE_FUNCTION_NAME = 'init_plugin'
PLUGIN_LOAD_TIME_WARN_LIMIT = 100  # milliseconds
PLUGIN_INIT_TIME__WARN_LIMIT = 100  # milliseconds

#: The extension point identifier for input processor extensions
EXTENSION_POINT_INPUT_PROCESSORS = 'xcube.core.gen.iproc'
#: The extension point identifier for dataset I/O extensions
EXTENSION_POINT_DATASET_IOS = 'xcube.core.dsio'
#: The extension point identifier for CLI command extensions
EXTENSION_POINT_CLI_COMMANDS = 'xcube.cli'
#: The extension point identifier for data stores
EXTENSION_POINT_DATA_STORES = 'xcube.core.store'
#: The extension point identifier for data openers
EXTENSION_POINT_DATA_OPENERS = 'xcube.core.store.opener'
#: The extension point identifier for data writers
EXTENSION_POINT_DATA_WRITERS = 'xcube.core.store.writer'

GLOBAL_GEO_EXTENT = -180., -90., 180., 90.

WGS84_ELLIPSOID_SEMI_MAJOR_AXIS = 6378137.
EARTH_EQUATORIAL_PERIMETER = 2. * math.pi * WGS84_ELLIPSOID_SEMI_MAJOR_AXIS

CRS_WKT_EPSG_4326 = """
GEOGCS["WGS 84",
    DATUM["WGS_1984",
        SPHEROID["WGS 84",6378137,298.257223563,
            AUTHORITY["EPSG","7030"]],
        AUTHORITY["EPSG","6326"]],
    PRIMEM["Greenwich",0,
        AUTHORITY["EPSG","8901"]],
    UNIT["degree",0.01745329251994328,
        AUTHORITY["EPSG","9122"]],
    AUTHORITY["EPSG","4326"]]
"""

FORMAT_NAME_ZARR = "zarr"
FORMAT_NAME_NETCDF4 = "netcdf4"
FORMAT_NAME_CSV = "csv"
FORMAT_NAME_MEM = "mem"
FORMAT_NAME_LEVELS = "levels"
FORMAT_NAME_SCRIPT = "script"
FORMAT_NAME_EXCEL = "excel"

# Note: this list must be kept in-sync with xcube.core.reproject:NAME_TO_GDAL_RESAMPLE_ALG
RESAMPLING_METHOD_NAMES = {
    # Up-sampling
    'Nearest',
    'Bilinear',
    'Cubic',
    'CubicSpline',
    'Lanczos',

    # Down-sampling
    'Average',
    'Min',
    'Max',
    'Median',
    'Mode',
    'Q1',
    'Q3',
}

# xcube's standard logger
LOG = logging.getLogger('xcube')


# xcube/plugin.py

# The MIT License (MIT)
# Copyright (c) 2020 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from xcube.constants import EXTENSION_POINT_CLI_COMMANDS
from xcube.constants import EXTENSION_POINT_DATASET_IOS
from xcube.constants import EXTENSION_POINT_DATA_OPENERS
from xcube.constants import EXTENSION_POINT_DATA_STORES
from xcube.constants import EXTENSION_POINT_DATA_WRITERS
from xcube.constants import EXTENSION_POINT_INPUT_PROCESSORS
from xcube.constants import FORMAT_NAME_CSV
from xcube.constants import FORMAT_NAME_MEM
from xcube.constants import FORMAT_NAME_NETCDF4
from xcube.constants import FORMAT_NAME_ZARR
from xcube.util import extension


def init_plugin(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard extensions.
    """
    _register_input_processors(ext_registry)
    _register_dataset_ios(ext_registry)
    _register_cli_commands(ext_registry)
    _register_data_stores(ext_registry)
    _register_data_accessors(ext_registry)


def _register_input_processors(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard input processors used by "xcube gen" and gen_cube().
    """
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.gen.iproc:DefaultInputProcessor'),
        point=EXTENSION_POINT_INPUT_PROCESSORS, name='default',
        description='Single-scene NetCDF/CF inputs in xcube standard format'
    )


def _register_dataset_ios(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard dataset I/O components used by various CLI and API functions.
    """
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.dsio:ZarrDatasetIO', call=True),
        point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_ZARR,
        description='Zarr file format (http://zarr.readthedocs.io)',
        ext='zarr', modes={'r', 'w', 'a'}
    )
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.dsio:Netcdf4DatasetIO', call=True),
        point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_NETCDF4,
        description='NetCDF-4 file format',
        ext='nc', modes={'r', 'w', 'a'}
    )
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.dsio:CsvDatasetIO', call=True),
        point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_CSV,
        description='CSV file format',
        ext='csv', modes={'r', 'w'}
    )
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.dsio:MemDatasetIO', call=True),
        point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_MEM,
        description='In-memory dataset I/O',
        ext='mem', modes={'r', 'w', 'a'}
    )


_FS_STORAGE_ITEMS = (
    ('file', 'local filesystem'),
    ('s3', 'AWS S3 compatible object storage'),
    ('memory', 'in-memory filesystem')
)

_FS_DATA_ACCESSOR_ITEMS = (
    ('dataset', 'netcdf',
     'xarray.Dataset in NetCDF format'),
    ('dataset', 'zarr',
     'xarray.Dataset in Zarr format'),
    ('dataset', 'levels',
     'xarray.Dataset in leveled Zarr format'),
    ('mldataset', 'levels',
     'xcube.core.mldataset.MultiLevelImage in leveled Zarr format'),
    ('geodataframe', 'shapefile',
     'gpd.GeoDataFrame in ESRI Shapefile format'),
    ('geodataframe', 'geojson',
     'gpd.GeoDataFrame in GeoJSON format'),
)

_FS_DATA_OPENER_ITEMS = _FS_DATA_ACCESSOR_ITEMS
_FS_DATA_WRITER_ITEMS = _FS_DATA_ACCESSOR_ITEMS


def _register_data_stores(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard data stores.
    """
    factory = 'xcube.core.store.fs.registry:get_fs_data_store_class'

    for storage_id, storage_description in _FS_STORAGE_ITEMS:
        loader = extension.import_component(factory, call_args=[storage_id])
        ext_registry.add_extension(
            point=EXTENSION_POINT_DATA_STORES,
            loader=loader,
            name=storage_id,
            description=f'Data store that uses a {storage_description}'
        )


def _register_data_accessors(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard data accessors.
    """
    factory = 'xcube.core.store.fs.registry:get_fs_data_accessor_class'

    # noinspection PyShadowingNames
    def _add_fs_data_accessor_ext(point: str,
                                  ext_type: str,
                                  protocol: str,
                                  data_type: str,
                                  format_id: str):
        factory_args = (protocol, data_type, format_id)
        loader = extension.import_component(factory,
                                            call_args=factory_args)
        ext_registry.add_extension(
            point=point,
            loader=loader,
            name=f'{data_type}:{format_id}:{protocol}',
            description=f'Data {ext_type} for'
                        f' a {data_accessor_description}'
                        f' in {storage_description}'
        )

    for protocol, storage_description in _FS_STORAGE_ITEMS:
        for data_type, format_id, data_accessor_description \
                in _FS_DATA_OPENER_ITEMS:
            _add_fs_data_accessor_ext(EXTENSION_POINT_DATA_OPENERS,
                                      'opener',
                                      protocol,
                                      data_type,
                                      format_id)
        for data_type, format_id, data_accessor_description \
                in _FS_DATA_WRITER_ITEMS:
            _add_fs_data_accessor_ext(EXTENSION_POINT_DATA_WRITERS,
                                      'writer',
                                      protocol,
                                      data_type,
                                      format_id)


def _register_cli_commands(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard CLI commands.
    """

    cli_command_names = [
        'chunk',
        'compute',
        'benchmark',
        'dump',
        'edit',
        'extract',
        'gen',
        'gen2',
        'genpts',
        'grid',
        'level',
        'optimize',
        'prune',
        'rectify',
        'resample',
        'serve',
        'tile',
        'vars2dim',
        'verify',
        'versions',

        # Experimental + Hidden
        'io',
    ]

    for cli_command_name in cli_command_names:
        ext_registry.add_extension(
            loader=extension.import_component(f'xcube.cli.{cli_command_name}:{cli_command_name}'),
            point=EXTENSION_POINT_CLI_COMMANDS,
            name=cli_command_name
        )


# xcube/version.py

# The MIT License (MIT)
# Copyright (c) 2021 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

version = '0.11.0.dev0'


# test/webapi/res/test/script.py

import numpy as np
import xarray as xr


def compute_dataset(ds, period='1W', incl_stdev=False):
    if incl_stdev:
        resample_obj = ds.resample(time=period)
        ds_mean = resample_obj.mean(dim='time')
        ds_std = resample_obj.std(dim='time').rename(name_dict={name: f"{name}_stdev" for name in ds.data_vars})
        ds_merged = xr.merge([ds_mean, ds_std])
        ds_merged.attrs.update(ds.attrs)
        return ds_merged
    else:
        return ds.resample(time=period).mean(dim='time')


def compute_variables(ds, factor_chl, factor_tsm):
    chl_tsm_sum = factor_chl * ds.conc_chl + factor_tsm * ds.conc_tsm
    chl_tsm_sum.attrs.update(dict(units='-',
                                  long_name='Weighted sum of CHL nd TSM concentrations',
                                  description='Nonsense variable, for demo purpose only'))

    chl_category = _categorize_chl(ds.conc_chl)
    chl_category.attrs.update(dict(units='-',
                                   long_name='Categorized CHL',
                                   description='0: 0<=CHL<3, 1: 3<=CHL<4, 2: CHL>4 mg/m^3'))

    return xr.Dataset(dict(chl_tsm_sum=chl_tsm_sum,
                           chl_category=chl_category))


def _categorize_chl(chl):
    return xr.where(chl >= 4., 2,
                    xr.where(chl >= 3.0, 1,
                             xr.where(chl >= 0.0, 0,
                                      np.nan)))


# test/cli/helpers.py

import unittest
from abc import ABCMeta
from typing import List, Dict, Optional

import click
import click.testing

from xcube.core.new import new_cube
from xcube.cli.main import cli
from xcube.core.dsio import rimraf

TEST_NC_FILE = "test.nc"
TEST_ZARR_DIR = "test.zarr"


class CliTest(unittest.TestCase, metaclass=ABCMeta):

    def invoke_cli(self, args: List[str]):
        self.runner = click.testing.CliRunner()
        return self.runner.invoke(cli, args, catch_exceptions=False)


class CliDataTest(CliTest, metaclass=ABCMeta):

    def outputs(self) -> List[str]:
        return []

    def time_periods(self) -> int:
        return 5

    def chunks(self) -> Optional[Dict[str, int]]:
        return None

    def setUp(self):
        self._rm_outputs()
        dataset = new_cube(variables=dict(precipitation=0.4, temperature=275.2, soil_moisture=0.5),
                           time_periods=self.time_periods())
        if self.chunks() is not None:
            dataset = dataset.chunk(self.chunks())
        dataset.to_netcdf(TEST_NC_FILE, mode="w")
        dataset.to_zarr(TEST_ZARR_DIR, mode="w")

    def tearDown(self):
        self._rm_outputs()

    def _rm_outputs(self):
        for path in [TEST_ZARR_DIR, TEST_NC_FILE] + self.outputs():
            rimraf(path)


# test/cli/test_benchmark.py

import os
import unittest

from test.cli.helpers import CliDataTest


@unittest.skip('Very slow tests, execute on demand only')
class BenchmarkCliTest(CliDataTest):

    def test_help_option(self):
        result = self.invoke_cli(['benchmark', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_simple(self):
        config_path = os.path.join(os.path.dirname(__file__), 'benchmark-configs', 'simple.yml')
        result = self.invoke_cli(['benchmark', config_path])
        msg = f'actual output:\n{result.stdout}'
        self.assertTrue('# command template: xcube dump ${input}\n' in result.stdout, msg=msg)
        self.assertTrue('\n# repetition count: 1\n' in result.stdout, msg=msg)
        self.assertTrue('\nid;input;time\n' in result.stdout, msg=msg)
        self.assertTrue('\n0;test.zarr;' in result.stdout, msg=msg)
        self.assertTrue('\n1;test.nc;' in result.stdout, msg=msg)

    def test_simple_with_repetitions(self):
        config_path = os.path.join(os.path.dirname(__file__), 'benchmark-configs', 'simple.yml')
        result = self.invoke_cli(['benchmark', '--repeats', 3, config_path])
        msg = f'actual output:\n{result.stdout}'

        self.assertTrue('# command template: xcube dump ${input}\n' in result.stdout, msg=msg)
        self.assertTrue('\n# repetition count: 3\n' in result.stdout, msg=msg)
        self.assertTrue('\nid;input;time-mean;time-median;time-stdev;time-min;time-max\n' in result.stdout, msg=msg)
        self.assertTrue('\n0;test.zarr;' in result.stdout, msg=msg)
        self.assertTrue('\n1;test.nc;' in result.stdout, msg=msg)


# test/cli/test_chunk.py

import os

import xarray as xr

from test.cli.helpers import CliDataTest, TEST_NC_FILE, TEST_ZARR_DIR


class ChunkTest(CliDataTest):
    TEST_OUTPUT = "test-chunked.zarr"

    def outputs(self):
        return [self.TEST_OUTPUT]

    def test_chunk_zarr(self):
        output_path = ChunkTest.TEST_OUTPUT
        result = self.invoke_cli(["chunk",
                                  TEST_ZARR_DIR,
                                  "-o", output_path,
                                  "--chunks", "time=1,lat=20,lon=40"])
        self.assertEqual("", result.output)
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(output_path))

        ds = xr.open_zarr(output_path)
        self.assertIn("precipitation", ds)
        precipitation = ds["precipitation"]
        self.assertTrue(hasattr(precipitation, "encoding"))
        self.assertIn("chunks", precipitation.encoding)
        self.assertEqual(precipitation.encoding["chunks"], (1, 20, 40))

    # TODO (forman): this test fails
    # netCDF4\_netCDF4.pyx:2437: in netCDF4._netCDF4.Dataset.createVariable
    # ValueError: cannot specify chunksizes for a contiguous dataset
    #
    # def test_chunk_nc(self):
    #     output_path = "test-chunked.nc"
    #     result = self.invoke_cli(["chunk",
    #                               TEST_NC_FILE,
    #                               "-o", output_path,
    #                               "--chunks", "time=1,lat=20,lon=40"])
    #     self.assertEqual("", result.output)
    #     self.assertEqual(0, result.exit_code)
    #     self.assertTrue(os.path.isdir(output_path))
    #     try:
    #         ds = xr.open_zarr(output_path)
    #         self.assertIn("precipitation", ds)
    #         precipitation = ds["precipitation"]
    #         self.assertTrue(hasattr(precipitation, "encoding"))
    #         self.assertIn("chunksizes", precipitation.encoding)
    #         self.assertEqual(precipitation.encoding["chunksizes"], (1, 20, 40))
    #     finally:
    #         os.remove(output_path)

    def test_chunk_size_syntax(self):
        result = self.invoke_cli(["chunk",
                                  TEST_NC_FILE,
                                  "-o", "test-chunked.zarr",
                                  "--chunks", "time=1,lat!gnnn,lon=40"])
        self.assertEqual("Error: Invalid value for CHUNKS:"
                         " 'time=1,lat!gnnn,lon=40'\n",
                         result.output)
        self.assertEqual(1, result.exit_code)

    def test_chunk_size_not_an_int(self):
        result = self.invoke_cli(["chunk",
                                  TEST_NC_FILE,
                                  "-o", "test-chunked.zarr",
                                  "--chunks", "time=1,lat=20.3,lon=40"])
        self.assertEqual("Error: Invalid value for CHUNKS,"
                         " chunk sizes must be positive integers:"
                         " time=1,lat=20.3,lon=40\n",
                         result.output)
        self.assertEqual(1, result.exit_code)

    def test_chunk_size_not_a_dim(self):
        result = self.invoke_cli(["chunk",
                                  TEST_NC_FILE,
                                  "-o", "test-chunked.zarr",
                                  "--chunks", "time=1,lati=20,lon=40"])
        self.assertEqual("Error: Invalid value for CHUNKS,"
                         " 'lati' is not the name of any dimension:"
                         " time=1,lati=20,lon=40\n",
                         result.output)
        self.assertEqual(1, result.exit_code)


# test/cli/test_common.py

import unittest

import click

from xcube.cli.common import assert_positive_int_item
from xcube.cli.common import handle_cli_exception
from xcube.cli.common import parse_cli_kwargs
from xcube.cli.common import parse_cli_sequence


class ClickUtilTest(unittest.TestCase):

    def test_parse_cli_kwargs(self):
        self.assertEqual(dict(),
                         parse_cli_kwargs("", metavar="<chunks>"))
        self.assertEqual(dict(time=1, lat=256, lon=512),
                         parse_cli_kwargs("time=1, lat=256, lon=512", metavar="<chunks>"))
        self.assertEqual(dict(chl_conc=(0, 20, 'greens'), chl_tsm=(0, 15, 'viridis')),
                         parse_cli_kwargs("chl_conc=(0,20,'greens'),chl_tsm=(0,15,'viridis')",
                                          metavar="<styles>"))

        with self.assertRaises(click.ClickException) as cm:
            parse_cli_kwargs("45 * 'A'", metavar="<chunks>")
        self.assertEqual("Invalid value for <chunks>: \"45 * 'A'\"",
                         f"{cm.exception}")

        with self.assertRaises(click.ClickException) as cm:
            parse_cli_kwargs("9==2")
        self.assertEqual("Invalid value: '9==2'",
                         f"{cm.exception}")

    def test_parse_cli_sequence(self):
        self.assertEqual((512, 1024),
                         parse_cli_sequence('512, 1024',
                                            metavar='X',
                                            item_parser=int))
        self.assertEqual((11, 11, 11),
                         parse_cli_sequence('11',
                                            metavar='X',
                                            item_parser=int,
                                            num_items=3))
        self.assertEqual((11, 12, 13),
                         parse_cli_sequence(['11', '12 ', ' 13'],
                                            metavar='X',
                                            item_parser=int))
        self.assertEqual(None,
                         parse_cli_sequence(None, allow_none=True))
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence(None,
                               metavar='X',
                               allow_none=False)
        self.assertEqual("X must be given", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11',
                               metavar='X',
                               item_parser=int,
                               num_items_min=3,
                               item_plural_name='lollies')
        self.assertEqual("X must have at least 3 lollies separated by ','", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11,12,13',
                               metavar='X',
                               item_parser=int,
                               num_items_max=2,
                               item_plural_name='lollies')
        self.assertEqual("X must have no more than 2 lollies separated by ','", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11,12,13',
                               metavar='X',
                               item_parser=int,
                               num_items=2,
                               item_plural_name='lollies')
        self.assertEqual("X must have 2 lollies separated by ','", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11,,13',
                               metavar='X',
                               item_parser=int,
                               num_items=3,
                               item_plural_name='lollies',
                               allow_empty_items=False)
        self.assertEqual("lollies in X must not be empty", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11,12,-13',
                               metavar='X',
                               item_parser=int,
                               item_validator=assert_positive_int_item,
                               num_items=3,
                               item_plural_name='lollies')
        self.assertEqual("Invalid lollies in X found: all items must be positive integer numbers", f'{cm.exception}')
        with self.assertRaises(ValueError) as cm:
            parse_cli_sequence('11,12,-13',
                               metavar='X',
                               item_parser=int,
                               item_validator=assert_positive_int_item,
                               num_items=3,
                               item_plural_name='lollies',
                               error_type=ValueError)
        self.assertEqual("Invalid lollies in X found: all items must be positive integer numbers", f'{cm.exception}')

    def test_handle_cli_exception(self):
        self.assertEqual(1, handle_cli_exception(click.Abort(), traceback_mode=True))
        self.assertEqual(1, handle_cli_exception(click.ClickException("bad handle"), traceback_mode=True))
        self.assertEqual(2, handle_cli_exception(OSError("bad handle"), traceback_mode=True))
        self.assertEqual(3, handle_cli_exception(ValueError("bad handle"), traceback_mode=True))
        self.assertEqual(20, handle_cli_exception(click.Abort(), exit_code=20))
        self.assertEqual(20, handle_cli_exception(click.ClickException("bad handle"), exit_code=20))
        self.assertEqual(20, handle_cli_exception(OSError("bad handle"), exit_code=20))
        self.assertEqual(20, handle_cli_exception(ValueError("bad handle"), exit_code=20))


# test/cli/test_compute.py

import os
from typing import List

import xarray as xr

from test.cli.helpers import CliDataTest, TEST_ZARR_DIR

OUTPUT_PATH = 'out.zarr'


class ComputeCliTest(CliDataTest):

    def outputs(self) -> List[str]:
        return [OUTPUT_PATH]

    def test_help_option(self):
        result = self.invoke_cli(['compute', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_compute_with_init_and_finalize(self):
        result = self.invoke_cli(['compute',
                                  os.path.join(os.path.dirname(__file__), 'compute-scripts', 'with-init-and-finalize.py'),
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(OUTPUT_PATH))

        ds = xr.open_zarr(OUTPUT_PATH)
        self.assertEqual(['output'], list(ds.data_vars))
        self.assertEqual('mg/m^3', ds.output.attrs.get('units'))
        self.assertEqual('I has a bucket', ds.attrs.get('comment'))
        self.assertAlmostEqual(0.2 * 0.4 + 0.3 * 0.5, float(ds.output.mean()))

    def test_compute_without_init(self):
        result = self.invoke_cli(['compute',
                                  '--params', 'a=0.1,b=0.4',
                                  '--vars', 'precipitation,soil_moisture',
                                  os.path.join(os.path.dirname(__file__), 'compute-scripts', 'without-init.py'),
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(OUTPUT_PATH))

        ds = xr.open_zarr(OUTPUT_PATH)
        self.assertEqual(['output'], list(ds.data_vars))
        self.assertAlmostEqual(0.1 * 0.4 + 0.4 * 0.5, float(ds.output.mean()))


# test/cli/test_dump.py

from test.cli.helpers import CliDataTest, TEST_NC_FILE
import xarray as xr


class DumpTest(CliDataTest):

    def test_dump_ds(self):
        self.maxDiff = None
        with xr.set_options(display_width=80):
            result = self.invoke_cli(["dump", TEST_NC_FILE])

        # xarray v0.16.0 changed line truncation behaviour (see
        # https://github.com/pydata/xarray/issues/3759 ), so this test checks
        # the output against two expected values to cover both pre- and
        # post-change behaviour.
        output_template = (
            "<xarray.Dataset>\n"
            "Dimensions:        (time: 5, lat: 180, lon: 360, bnds: 2)\n"
            "Coordinates:\n"
            "  * lon            (lon) float64 -179.5 -178.5 -177.5 ... 177.5 178.5 179.5\n"
            "  * lat            (lat) float64 -89.5 -88.5 -87.5 -86.5 ... 86.5 87.5 88.5 89.5\n"
            "  * time           (time) datetime64[ns] 2010-01-01T12:00:00 ... 2010-01-05T1%s\n"
            "    lon_bnds       (lon, bnds) float64 ...\n"
            "    lat_bnds       (lat, bnds) float64 ...\n"
            "    time_bnds      (time, bnds) datetime64[ns] ...\n"
            "Dimensions without coordinates: bnds\n"
            "Data variables:\n"
            "    precipitation  (time, lat, lon) float64 ...\n"
            "    temperature    (time, lat, lon) float64 ...\n"
            "    soil_moisture  (time, lat, lon) float64 ...\n"
            "Attributes:\n"
            "    Conventions:           CF-1.7\n"
            "    title:                 Test Cube\n"
            "    time_coverage_start:   2010-01-01T00:00:00\n"
            "    time_coverage_end:     2010-01-06T00:00:00\n"
            "    geospatial_lon_min:    -180.0\n"
            "    geospatial_lon_max:    180.0\n"
            "    geospatial_lon_units:  degrees_east\n"
            "    geospatial_lat_min:    -90.0\n"
            "    geospatial_lat_max:    90.0\n"
            "    geospatial_lat_units:  degrees_north\n"
        )

        possible_outputs = [
            output_template % '2:00:00',  # for xarray v0.15.1 and below
            output_template % '...'  # for xarray v0.16.0 and above
        ]
        print(result.output)
        self.assertIn(result.output, possible_outputs)
        self.assertEqual(0, result.exit_code)


# test/cli/test_edit.py

import yaml
import zarr

from test.cli.helpers import CliDataTest
from test.core.test_edit import TEST_CUBE, TEST_CUBE_ZARR, TEST_CUBE_ZARR_EDIT, TEST_NEW_META, TEST_NEW_META_YML, \
    TEST_CUBE_ZARR_COORDS, TEST_CUBE_COORDS
from xcube.core.dsio import rimraf

TEST_CUBE_ZARR_EDIT_DEFAULT = 'test-edited.zarr'


class EditMetadaDataTest(CliDataTest):
    def _clear_outputs(self):
        rimraf(TEST_CUBE_ZARR)
        rimraf(TEST_CUBE_ZARR_EDIT)
        rimraf(TEST_NEW_META_YML)
        rimraf(TEST_CUBE_ZARR_EDIT_DEFAULT)
        rimraf(TEST_CUBE_ZARR_COORDS)

    def setUp(self):
        self._clear_outputs()
        TEST_CUBE.to_zarr(TEST_CUBE_ZARR)
        TEST_CUBE_COORDS.to_zarr(TEST_CUBE_ZARR_COORDS)
        with open(TEST_NEW_META_YML, 'w') as outfile:
            yaml.dump(TEST_NEW_META, outfile, default_flow_style=False)

    def tearDown(self):
        self._clear_outputs()

    def test_defaults(self):
        result = self.invoke_cli(['edit', TEST_CUBE_ZARR, '-M', TEST_NEW_META_YML])
        self.assertEqual(0, result.exit_code)

        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT_DEFAULT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertNotIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_name', ds2.attrs.keys())

    def test_user_output(self):
        result = self.invoke_cli(['edit', TEST_CUBE_ZARR, '-M', TEST_NEW_META_YML, '-o', TEST_CUBE_ZARR_EDIT])
        self.assertEqual(0, result.exit_code)

        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertNotIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_name', ds2.attrs.keys())

    def test_in_place(self):
        result = self.invoke_cli(['edit', '-I', TEST_CUBE_ZARR, '-M', TEST_NEW_META_YML])
        self.assertEqual(0, result.exit_code)

        ds1 = zarr.open(TEST_CUBE_ZARR)
        self.assertEqual(36, ds1.__len__())
        self.assertEqual('14-APR-2017 10:27:50.183264', ds1.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds1['conc_chl'].attrs.__getitem__('units'))
        self.assertIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_url', ds1.attrs.keys())
        self.assertEqual('Brockmann Consult GmbH with love', ds1.attrs.__getitem__('creator_name'))
        self.assertEqual('www.some_very_nice_url.com', ds1.attrs.__getitem__('creator_url'))

    def test_update_coords_only(self):
        ds1 = zarr.open(TEST_CUBE_ZARR_COORDS)
        delete_list = ['geospatial_lat_max', 'geospatial_lat_min', 'geospatial_lat_units', 'geospatial_lon_max',
                       'geospatial_lon_min', 'geospatial_lon_units', 'time_coverage_end', 'time_coverage_start']
        for attr in ds1.attrs.keys():
            if attr in delete_list:
                ds1.attrs.__delitem__(attr)
        result = self.invoke_cli(['edit', TEST_CUBE_ZARR_COORDS, '-o', TEST_CUBE_ZARR_EDIT, '-C'])
        self.assertEqual(0, result.exit_code)

        ds1 = zarr.open(TEST_CUBE_ZARR_COORDS)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        for attr in delete_list:
            self.assertNotIn(attr, ds1.attrs.keys())
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertIn('geospatial_lat_max', ds2.attrs.keys())
        self.assertIn('geospatial_lat_min', ds2.attrs.keys())
        self.assertIn('geospatial_lat_units', ds2.attrs.keys())
        self.assertIn('geospatial_lon_max', ds2.attrs.keys())
        self.assertEqual(180.0, ds2.attrs.__getitem__('geospatial_lon_max'))
        self.assertEqual(-180.0, ds2.attrs.__getitem__('geospatial_lon_min'))
        self.assertEqual('2010-01-04T00:00:00', ds2.attrs.__getitem__('time_coverage_end'))
        self.assertEqual('2010-01-01T00:00:00', ds2.attrs.__getitem__('time_coverage_start'))
        self.assertEqual('degrees_east', ds2.attrs.__getitem__('geospatial_lon_units'))


class EditTest(CliDataTest):

    def test_help_option(self):
        result = self.invoke_cli(['edit', '--help'])
        self.assertEqual(0, result.exit_code)


# test/cli/test_extract.py

import os
import os.path
import warnings
from io import StringIO

import pandas as pd

from test.cli.helpers import CliDataTest

VAR_COLS = [
    'c2rcc_flags',
    'conc_chl',
    'conc_tsm',
    'kd489',
    'quality_flags',
]

COORDS_COLS = [
    'lon',
    'lat',
    'time',
]

BOUNDS_COLS = [
    'lon_lower',
    'lon_upper',
    'lat_lower',
    'lat_upper',
    'time_lower',
    'time_upper',
]

INDEX_COLS = [
    'lon_index',
    'lat_index',
    'time_index',
]

REF_COLS = [
    'id_ref',
    'lon_ref',
    'lat_ref',
    'time_ref',
]


class ExtractCliTest(CliDataTest):

    @classmethod
    def setUpClass(cls):
        # We parse CSV-output from CLI using pandas in these tests.
        # Warnings will cause the pandas parser to fail.
        warnings.filterwarnings("ignore")

    @classmethod
    def tearDownClass(cls):
        warnings.resetwarnings()

    def test_help_option(self):
        result = self.invoke_cli(['extract', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_all_invalid_points(self):
        self._do_test_points('all-invalid-points.csv')

    def test_some_invalid_points(self):
        self._do_test_points('some-invalid-points.csv')

    def test_extract_points_with_refs(self):
        self._do_test_points('all-valid-points.csv')

    def _do_test_points(self, filename):
        self._do_test_points_with_options(filename, options=[],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--coords'],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--bounds'],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--refs'],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--indexes'],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--coords', '--bounds', '--refs', '--indexes'],
                                          expected_length=6)

    def _do_test_points_with_options(self,
                                     filename,
                                     options=None,
                                     expected_length=0):
        base_dir = os.path.dirname(__file__)
        args = ['extract']
        if options:
            args += options
        args += [os.path.join(base_dir, '..', '..', 'examples', 'serve', 'demo', 'cube.nc'),
                 os.path.join(base_dir, 'extract-points', filename)]
        result = self.invoke_cli(args)
        self.assertEqual(0, result.exit_code)
        self.assertNotEqual('', result.stdout)
        try:
            df = pd.read_csv(StringIO(result.stdout),
                             index_col='idx')
        except pd.errors.ParserError as e:
            print(f"Failed to parse result: {e}:")
            print(result.stdout)
            raise e
        self.assertEqual(expected_length, len(df))
        expected_columns = set(VAR_COLS)
        if '--coords' in options:
            expected_columns.update(COORDS_COLS)
        if '--bounds' in options:
            expected_columns.update(BOUNDS_COLS)
        if '--refs' in options:
            expected_columns.update(REF_COLS)
        if '--indexes' in options:
            expected_columns.update(INDEX_COLS)
        self.assertEqual(expected_columns, set(df.columns))


# test/cli/test_gen.py

from test.cli.helpers import CliTest


class GenCliTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['gen', 'time_irregular', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_help(self):
        result = self.invoke_cli(['gen', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_info(self):
        result = self.invoke_cli(['gen', '--info'])
        self.assertEqual(0, result.exit_code)

    def test_default_proc_is_valid(self):
        result = self.invoke_cli(['gen', '-P', 'default', 'input.nc'])
        self.assertEqual(0, result.exit_code)

    def test_main_with_illegal_proc(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '--proc', 'gnartz'])
        self.assertEqual("Unknown input_processor_name 'gnartz'", f'{cm.exception}')

    def test_main_with_illegal_size_option(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '-S', '120,abc', 'input.nc'])
        self.assertEqual("output_size must have the form <width>,<height>,"
                         " where both values must be positive integer numbers", f'{cm.exception}')

    def test_main_with_illegal_region_option(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '-R', '50,_2,55,21', 'input.nc'])
        self.assertEqual("output_region must have the form <lon_min>,<lat_min>,<lon_max>,<lat_max>,"
                         " where all four numbers must be floating point numbers in degrees", f'{cm.exception}')
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '-R', '50,20,55', 'input.nc'])
        self.assertEqual("output_region must have the form <lon_min>,<lat_min>,<lon_max>,<lat_max>,"
                         " where all four numbers must be floating point numbers in degrees", f'{cm.exception}')

    def test_main_with_illegal_variable_option(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '--vars', ' ', 'input.nc'])
        self.assertEqual("output_variables must be a list of existing variable names", f'{cm.exception}')

    def test_main_with_illegal_config_option(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '-c', 'nonono.yml', 'input.nc'])
        self.assertEqual("Cannot find configuration 'nonono.yml'", f'{cm.exception}')




# test/cli/test_gen2.py

import json
import os
import os.path

from test.cli.helpers import CliTest
from xcube.core.dsio import rimraf

result_file = 'result.json'
result_zarr = 'out.zarr'
result_levels = 'out.levels'


class Gen2CliTest(CliTest):

    def setUp(self) -> None:
        rimraf(result_file, result_zarr, result_levels)

    def tearDown(self) -> None:
        rimraf(result_file, result_zarr, result_levels)

    def read_result_json(self):
        self.assertTrue(os.path.exists(result_file))
        with open(result_file) as fp:
            result_json = json.load(fp)
        self.assertIsInstance(result_json, dict)
        return result_json

    def test_help(self):
        result = self.invoke_cli(['gen2', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_copy_zarr_gen(self):
        request_file = os.path.join(os.path.dirname(__file__),
                                    'gen2-requests', 'copy-zarr.yml')
        result = self.invoke_cli(['gen2',
                                  '-o', result_file,
                                  request_file])
        self.assertIsNotNone(result)
        result_json = self.read_result_json()
        self.assertEqual('ok', result_json.get('status'))
        self.assertEqual(201, result_json.get('status_code'))
        self.assertIsInstance(result_json.get('message'), str)
        self.assertIn('Cube generated successfully after ',
                      result_json.get('message'))
        result = result_json.get('result')
        self.assertIsInstance(result, dict)
        self.assertEqual(result_zarr, result.get('data_id'))
        self.assertTrue(os.path.isdir(result_zarr))

    def test_copy_zarr_info(self):
        request_file = os.path.join(os.path.dirname(__file__),
                                    'gen2-requests', 'copy-zarr.yml')
        result = self.invoke_cli(['gen2',
                                  '--info',
                                  '-o', result_file,
                                  request_file])
        self.assertIsNotNone(result)
        result_json = self.read_result_json()

        self.assertEqual('ok', result_json.get('status'))
        self.assertEqual(200, result_json.get('status_code'))
        result = result_json.get('result')
        self.assertIsInstance(result, dict)

        self.assertEqual(
            {
                'dataset_descriptor': {
                    'data_id': result_zarr,
                    'data_type': 'dataset',
                    'dims': {'lat': 1000, 'lon': 2000, 'time': 15},
                    'spatial_res': 0.0024999999999977263,
                    'time_period': '1D',
                    'time_range': ['2017-01-16', '2017-01-30'],
                    'bbox': [0.0, 50.0, 5.0, 52.5],
                    'crs': 'WGS84',
                    'data_vars': {
                        'c2rcc_flags': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'c2rcc_flags'
                        },
                        'conc_chl': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'conc_chl'
                        },
                        'conc_tsm': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'conc_tsm'
                        },
                        'kd489': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'kd489'
                        },
                        'quality_flags': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'quality_flags'
                        }
                    },
                },
                'size_estimation': {
                    'image_size': [2000, 1000],
                    'num_bytes': 600000000,
                    'num_requests': 75,
                    'num_tiles': [1, 1],
                    'num_variables': 5,
                    'tile_size': [2000, 1000]
                }
            },
            result)
        self.assertFalse(os.path.isdir(result_zarr))

    def test_copy_levels_gen(self):
        request_file = os.path.join(os.path.dirname(__file__),
                                    'gen2-requests', 'copy-levels.yml')
        result = self.invoke_cli(['gen2',
                                  '-o', result_file,
                                  request_file])
        print(result.output)
        self.assertIsNotNone(result)
        result_json = self.read_result_json()
        self.assertEqual('ok', result_json.get('status'))
        self.assertEqual(201, result_json.get('status_code'))
        self.assertEqual({'data_id': 'out.levels'}, result_json.get('result'))
        self.assertTrue(os.path.isdir(result_levels))

    def test_internal_error(self):
        request_file = os.path.join(os.path.dirname(__file__),
                                    'gen2-requests', 'internal-error.yml')
        result = self.invoke_cli(['gen2',
                                  '-o', result_file,
                                  request_file])
        print(result.output)
        self.assertIsNotNone(result)
        result_json = self.read_result_json()
        self.assertEqual('error', result_json.get('status'))
        self.assertEqual(500, result_json.get('status_code'))
        self.assertEqual('inverse_fine_structure_constant must be 137'
                         ' or running in wrong universe',
                         result_json.get('message'))
        self.assertNotIn('result', result_json)


# test/cli/test_genpts.py

import os.path
from typing import List

import fiona
import pandas as pd

from test.cli.helpers import CliDataTest
from test.cli.helpers import TEST_ZARR_DIR

TEST_CSV = os.path.join(os.path.dirname(__file__), 'out.csv')
TEST_GEOJSON = os.path.join(os.path.dirname(__file__), 'out.geojson')


class GenptsCliTest(CliDataTest):

    def outputs(self) -> List[str]:
        return [TEST_CSV, TEST_GEOJSON]

    def test_help_option(self):
        result = self.invoke_cli(['extract', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_csv(self):
        result = self.invoke_cli(['genpts', TEST_ZARR_DIR, '--output', TEST_CSV])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('', result.stdout)
        df = pd.read_csv(TEST_CSV)
        self.assertEqual(100, len(df))

    def test_geojson(self):
        result = self.invoke_cli(['genpts', TEST_ZARR_DIR, '--output', TEST_GEOJSON])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('', result.stdout)
        with fiona.open(TEST_GEOJSON) as collection:
            features = [feature for feature in collection]
        self.assertEqual(100, len(features))


# test/cli/test_grid.py

import unittest
from fractions import Fraction
from typing import List

import click
import click.testing

from xcube.cli.grid import grid as cli
from xcube.cli.grid import factor_out_two, get_adjusted_box, get_levels, meters_to_degrees, find_close_resolutions


class GridToolsTest(unittest.TestCase):

    def test_factor_out_two(self):
        with self.assertRaises(ValueError):
            factor_out_two(-1)
        self.assertEqual((0, 0), factor_out_two(0))
        self.assertEqual((1, 0), factor_out_two(1))
        self.assertEqual((1, 1), factor_out_two(2))
        self.assertEqual((3, 0), factor_out_two(3))
        self.assertEqual((1, 2), factor_out_two(4))
        self.assertEqual((83743, 0), factor_out_two(83743))
        self.assertEqual((2617, 5), factor_out_two(83744))
        self.assertEqual(83744, 2617 * 2 ** 5)

    def test_get_adjusted_box(self):
        self.assertEqual((0.0, 49.21875, 5.625, 53.4375),
                         get_adjusted_box(0.0, 50.0, 5.0, 52.5, 540 / 384))

    def test_get_levels(self):
        results = get_levels(3, 180, level_min=4)
        self.assertEqual(6, len(results))
        self.assertEqual(['L', 0, 1, 2, 3, 4],
                         [row[0] for row in results])
        self.assertEqual(['H', 3, 6, 12, 24, 48],
                         [row[1] for row in results])
        self.assertEqual(['R', Fraction(60, 1), Fraction(30, 1), Fraction(15, 1), Fraction(15, 2), Fraction(15, 4)],
                         [row[2] for row in results])
        self.assertEqual(['R (deg)', 60.0, 30.0, 15.0, 7.5, 3.75],
                         [row[3] for row in results])
        self.assertEqual(['R (m)', 6679169.45, 3339584.72, 1669792.36, 834896.18, 417448.09],
                         [row[4] for row in results])

    def test_find_close_resolutions(self):
        target_res = meters_to_degrees(300)
        results = find_close_resolutions(target_res, target_res * 1.0 / 100, 180)
        self.assertEqual(43, len(results))
        self.assertEqual(['R_D (%)', 0.012, -0.036, 0.059, -0.084],
                         [row[0] for row in results[0:5]])
        self.assertEqual(['R_NOM', 45, 5, 45, 45],
                         [row[1] for row in results[0:5]])
        self.assertEqual(['R_DEN', 16696, 1856, 16688, 16712],
                         [row[2] for row in results[0:5]])
        self.assertEqual(['R (deg)',
                          0.002695256348826066,
                          0.0026939655172413795,
                          0.002696548418024928,
                          0.0026926759214935376],
                         [row[3] for row in results[0:5]])
        self.assertEqual(['R (m)', 300.03, 299.89, 300.18, 299.75],
                         [row[4] for row in results[0:5]])
        self.assertEqual(['H', 66784, 66816, 66752, 66848],
                         [row[5] for row in results[0:5]])
        self.assertEqual(['H0', 2087, 261, 1043, 2089],
                         [row[6] for row in results[0:5]])


class GridCliTest(unittest.TestCase):

    @classmethod
    def invoke_cli(cls, args: List[str]):
        runner = click.testing.CliRunner()
        return runner.invoke(cli, args)

    def test_levels(self):
        result = self.invoke_cli(["levels", "-R", "1/384"])
        self.assertEqual(("""
L	H	R	R (deg)	R (m)
0	135	4/3	1.3333333333333333	148425.99
1	270	2/3	0.6666666666666666	74212.99
2	540	1/3	0.3333333333333333	37106.5
3	1080	1/6	0.16666666666666666	18553.25
4	2160	1/12	0.08333333333333333	9276.62
5	4320	1/24	0.041666666666666664	4638.31
6	8640	1/48	0.020833333333333332	2319.16
7	17280	1/96	0.010416666666666666	1159.58
8	34560	1/192	0.005208333333333333	579.79
9	69120	1/384	0.0026041666666666665	289.89
"""),
                         result.output)
        self.assertEqual(0, result.exit_code)

    def test_res(self):
        result = self.invoke_cli(["res", "300m", "-D", "1%", "--sort_by", "+H0", "-N", "6"])
        print(result.output)
        self.assertEqual(("""
R_D (%)	R_NOM	R_DEN	R (deg)	R (m)	H	H0	L
0.348	9	3328	0.002704326923076923	301.04	66560	65	10
-0.418	45	16768	0.0026836832061068704	298.75	67072	131	9
0.736	45	16576	0.00271476833976834	302.21	66304	259	8
-0.036	5	1856	0.0026939655172413795	299.89	66816	261	8
-0.797	45	16832	0.0026734790874524714	297.61	67328	263	8
37 more...
"""),
                         result.output)
        self.assertEqual(0, result.exit_code)

    def test_abox(self):
        result = self.invoke_cli(['abox', "0,50,5,52.5", "-R", "1/384"])
        self.assertEqual(("""
Orig. box coord. = 0.0,50.0,5.0,52.5
Adj. box coord.  = 0.0,49.33333333333333,5.333333333333333,53.33333333333333
Orig. box WKT    = POLYGON ((0.0 50.0, 5.0 50.0, 5.0 52.5, 0.0 52.5, 0.0 50.0))
Adj. box WKT     = POLYGON ((0.0 49.33333333333333, 5.333333333333333 49.33333333333333, 5.333333333333333 53.33333333333333, 0.0 53.33333333333333, 0.0 49.33333333333333))
Combined WKT     = MULTIPOLYGON (((0.0 50.0, 5.0 50.0, 5.0 52.5, 0.0 52.5, 0.0 50.0)), ((0.0 49.33333333333333, 5.333333333333333 49.33333333333333, 5.333333333333333 53.33333333333333, 0.0 53.33333333333333, 0.0 49.33333333333333)))
Box grid size    = 2048 x 1536 cells
Graticule dist.  = 4/3 degrees
Tile size        = 1 cells
Granularity      = 135 cells
Level            = 9
Res. at level 0  = 4/3 degrees
Resolution       = 1/384 degrees
                 = 289.89 meters
"""),
                         result.output)
        self.assertEqual(0, result.exit_code)


# test/cli/test_io.py

import json
import os.path

import yaml

from test.cli.helpers import CliTest
from xcube.core.dsio import rimraf


class IOTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['io', '--help'])
        self.assertEqual(0, result.exit_code)


class IOStoreTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['io', 'store', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'store', 'list', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'store', 'info', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_list(self):
        result = self.invoke_cli(['io', 'store', 'list'])
        self.assertEqual(0, result.exit_code)

    def test_info(self):
        result = self.invoke_cli(['io', 'store', 'info', 'memory',
                                  '-POWj'])
        self.assertEqual(0, result.exit_code)
        result = self.invoke_cli(['io', 'store', 'info', 'file',
                                  '-POW', 'root=.'])
        self.assertEqual(0, result.exit_code)

        # param "root" missing
        result = self.invoke_cli(['io', 'store', 'info',
                                  'file', '-POW', 'root=42'])
        self.assertEqual(1, result.exit_code)
        self.assertEqual("Error: Failed to instantiate data store 'file'"
                         " with parameters ('root=42',):"
                         " Invalid parameterization detected:"
                         " 42 is not of type 'string'\n",
                         result.stdout)

    def test_data(self):
        demo_dir = os.path.join(os.path.dirname(__file__),
                                '..', '..',
                                'examples', 'serve', 'demo')

        result = self.invoke_cli(['io', 'store', 'info',
                                  'file', '-D', f'root={demo_dir}'])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('\n'
                         'Data store description:\n'
                         '  Data store that uses a local filesystem\n'
                         '\n'
                         'Data resources:\n'
                         '             cube-1-250-250.levels  <no title>\n'
                         '               cube-1-250-250.zarr  <no title>\n'
                         '               cube-5-100-200.zarr  <no title>\n'
                         '                           cube.nc  <no title>\n'
                         '4 data resources found.\n',
                         result.stdout)

        demo_dir = os.path.join(os.path.dirname(__file__),
                                '..', '..',
                                'examples', 'serve', 'demo')
        result = self.invoke_cli(['io', 'store', 'data',
                                  'file', 'cube-1-250-250.zarr',
                                  f'root={demo_dir}'])
        self.assertEqual(0, result.exit_code)
        self.assertIn('cube-1-250-250.zarr', result.stdout)


class IOOpenerTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['io', 'opener', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'opener', 'list', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'opener', 'info', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_list(self):
        result = self.invoke_cli(['io', 'opener', 'list'])
        self.assertEqual(0, result.exit_code)

    def test_info(self):
        result = self.invoke_cli(['io', 'opener', 'info', 'dataset:zarr:file'])
        self.assertEqual(0, result.exit_code)


class IOWriterTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['io', 'writer', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'writer', 'list', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'writer', 'info', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_list(self):
        result = self.invoke_cli(['io', 'writer', 'list'])
        self.assertEqual(0, result.exit_code)

    def test_info(self):
        result = self.invoke_cli(['io', 'writer', 'info', 'dataset:zarr:file'])
        self.assertEqual(0, result.exit_code)


class IODumpTest(CliTest):
    STORE_CONF = {
        "this_dir": {
            "store_id": "file",
            "store_params": {
                "root": os.path.join(os.path.dirname(__file__),
                                     "..", "..",
                                     "examples", "serve", "demo")
            }
        }
    }

    @classmethod
    def setUpClass(cls) -> None:
        with open('store-conf.yml', 'w') as fp:
            yaml.dump(cls.STORE_CONF, fp)
        with open('store-conf.json', 'w') as fp:
            json.dump(cls.STORE_CONF, fp)

    @classmethod
    def tearDownClass(cls) -> None:
        rimraf('store-conf.json')
        rimraf('store-conf.yml')

    def tearDown(self) -> None:
        rimraf('store-dump.json')
        rimraf('store-dump.yml')
        rimraf('store-dump.csv')
        rimraf('out.json')
        rimraf('out.jml')
        rimraf('out.txt')

    def test_help_option(self):
        result = self.invoke_cli(['io', 'dump',
                                  '--help'])
        self.assertEqual(0, result.exit_code)

    def test_yaml_config(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.json'))

    def test_yaml_config_short_format(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml',
                                  '--short', '--json'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.json'))
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml',
                                  '--short', '--yaml'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.yml'))
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml',
                                  '--short', '--csv'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.csv'))

    def test_json_config_short(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml',
                                  '--short'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.json'))

    def test_json_config_output(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.json',
                                  '-o', 'out.json'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('out.json'))
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.json',
                                  '-o', 'out.yml'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('out.yml'))
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.json',
                                  '-o', 'out.txt'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('out.txt'))

    def test_json_config_type(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.json',
                                  '-t', 'dataset'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.json'))


# test/cli/test_level.py

import os.path
from typing import List, Tuple, Optional, Dict

from test.cli.helpers import CliDataTest
from test.cli.helpers import CliTest
from test.cli.helpers import TEST_NC_FILE, TEST_ZARR_DIR
from xcube.core.mldataset import MultiLevelDataset
from xcube.core.store import new_fs_data_store


class LevelTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['level', '--help'])
        self.assertEqual(0, result.exit_code)


class LevelDataTest(CliDataTest):
    TEST_OUTPUT = "test.levels"

    def outputs(self) -> List[str]:
        return [LevelDataTest.TEST_OUTPUT, 'my.levels']

    def chunks(self) -> Optional[Dict[str, int]]:
        return dict(time=1, lat=90, lon=180)

    def test_all_defaults(self):
        result = self.invoke_cli(['level', TEST_ZARR_DIR])
        self._assert_result_ok(
            result, [((1, 1, 1, 1, 1), (90, 90), (180, 180)),
                     ((1, 1, 1, 1, 1), (90,), (180,))],
            LevelDataTest.TEST_OUTPUT,
            'Multi-level dataset written to /test.levels after .*\n'
        )

    def test_with_output(self):
        result = self.invoke_cli(['level', TEST_ZARR_DIR,
                                  '--output', 'my.levels'])
        self._assert_result_ok(
            result, [((1, 1, 1, 1, 1), (90, 90), (180, 180)),
                     ((1, 1, 1, 1, 1), (90,), (180,))], 'my.levels',
            'Multi-level dataset written to my.levels after .*\n'
        )

    def test_with_tile_size_and_num_levels(self):
        result = self.invoke_cli(['level', TEST_ZARR_DIR,
                                  '-t', '90,45', '-n', '4'])
        self._assert_result_ok(
            result, [((1, 1, 1, 1, 1), (45, 45, 45, 45), (90, 90, 90, 90)),
                     ((1, 1, 1, 1, 1), (45, 45), (90, 90)),
                     ((1, 1, 1, 1, 1), (45,), (90,))],
            LevelDataTest.TEST_OUTPUT,
            'Multi-level dataset written to /test.levels after .*\n'
        )

    def _assert_result_ok(self,
                          result,
                          level_chunks: List[Tuple],
                          output_path: str,
                          message_regex: str):
        self.assertEqual(0, result.exit_code)
        self.assertRegex(result.stdout, message_regex)
        self.assertTrue(os.path.isdir(output_path))
        data_store = new_fs_data_store("file")
        ml_dataset = data_store.open_data(output_path)
        self.assertIsInstance(ml_dataset, MultiLevelDataset)
        self.assertEqual(len(level_chunks), ml_dataset.num_levels)
        for level in range(ml_dataset.num_levels):
            level_dataset = ml_dataset.get_dataset(level)
            self.assertEqual({'precipitation',
                              'soil_moisture',
                              'temperature'},
                             set(level_dataset.data_vars.keys()))
            for var_name, var in level_dataset.data_vars.items():
                var_chunks = level_chunks[level]
                self.assertEqual(var_chunks,
                                 var.chunks,
                                 f'{var_name} at level {level}')

    def _assert_result_not_ok(self, result, message_regex: str):
        self.assertEqual(1, result.exit_code)
        self.assertRegex(result.stdout, message_regex)

    def test_level_with_nc(self):
        result = self.invoke_cli(["level",
                                  "-t", "45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr(self):
        result = self.invoke_cli(["level",
                                  "-t", "45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_ZARR_DIR,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr_agg_method(self):
        result = self.invoke_cli(["level",
                                  "-t", "45",
                                  "-A", "auto",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_ZARR_DIR,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr_agg_methods(self):
        result = self.invoke_cli([
            "level",
            "-t", "45",
            "-A", "precipitation=mean,temperature=max,soil_*=median",
            "-o", LevelDataTest.TEST_OUTPUT,
            TEST_ZARR_DIR,
        ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr_link(self):
        result = self.invoke_cli(["level",
                                  "--link",
                                  "-t", "45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_ZARR_DIR,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.link', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr_num_levels_max(self):
        result = self.invoke_cli(["level",
                                  "-t", "45",
                                  "-n", "2",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_ZARR_DIR,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_invalid_inputs(self):
        result = self.invoke_cli(["level",
                                  "-t", "a45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE,
                                  ])
        self._assert_result_not_ok(
            result,
            "Error\\: Invalid tile sizes in TILE_SIZE found: "
            "invalid literal for int\\(\\) with base 10\\: 'a45'\n"
        )

        result = self.invoke_cli(["level",
                                  "-t", "-3",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE,
                                  ])
        self._assert_result_not_ok(
            result,
            "Error\\: Invalid tile sizes in TILE_SIZE found\\: "
            "all items must be positive integer numbers\n"
        )

        result = self.invoke_cli(["level",
                                  "-t", "45,45,45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE])
        self._assert_result_not_ok(
            result,
            "Error\\: TILE_SIZE must have 2 tile sizes separated by ','\n"
        )

        result = self.invoke_cli(["level",
                                  "-n", "0",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE])
        self._assert_result_not_ok(
            result,
            "NUM_LEVELS_MAX must be a positive integer\n"
        )

    def test_with_existing_output(self):
        # Product output
        self.invoke_cli(['level', TEST_ZARR_DIR,
                         '--output', 'my.levels'])
        # Product output once mor
        result = self.invoke_cli(['level', TEST_ZARR_DIR,
                                  '--output', 'my.levels'])
        self._assert_result_not_ok(
            result,
            'Error: output \'my\\.levels\' already exists\n'
        )


# test/cli/test_main.py

import warnings

import pytest
from click import command

import xcube.cli.main
from test.cli.helpers import CliTest


class ClickMainTest(CliTest):
    USER_WARNING_MESSAGE = 'This is a user warning.'
    DEPRECATION_WARNING_MESSAGE = 'This is a deprecation warning.'
    RUNTIME_WARNING_MESSAGE = 'This is a runtime warning.'

    def setUp(self):
        @command(name='test_warnings')
        def test_warnings():
            warnings.warn(self.USER_WARNING_MESSAGE)
            warnings.warn(self.DEPRECATION_WARNING_MESSAGE, category=DeprecationWarning)
            warnings.warn(self.RUNTIME_WARNING_MESSAGE, category=RuntimeWarning)

        xcube.cli.main.cli.add_command(test_warnings)

    def test_warnings_not_issued_by_default(self):
        with pytest.warns(None) as record:
            self.invoke_cli(['test_warnings'])
        self.assertEqual(['This is a user warning.'],
                         list(map(lambda r: str(r.message), record.list)))

    def test_warnings_issued_when_enabled(self):
        with pytest.warns(None) as record:
            self.invoke_cli(['--warnings', 'test_warnings'])
        self.assertEqual(['This is a user warning.',
                          'This is a deprecation warning.',
                          'This is a runtime warning.'],
                         list(map(lambda r: str(r.message), record.list)))


# test/cli/test_optimize.py

from test.cli.helpers import CliDataTest
from test.core.test_optimize import TEST_CUBE, INPUT_CUBE_PATH, INPUT_CUBE_FILE_SET, list_file_set
from xcube.core.dsio import rimraf

OUTPUT_CUBE_OPTIMIZED_DEFAULT_PATH = 'test-optimized.zarr'
OUTPUT_CUBE_OPTIMIZED_USER_PATH = 'fast-test.zarr'


class OptimizeDataTest(CliDataTest):
    def _clear_outputs(self):
        rimraf(INPUT_CUBE_PATH)
        rimraf(OUTPUT_CUBE_OPTIMIZED_DEFAULT_PATH)
        rimraf(OUTPUT_CUBE_OPTIMIZED_USER_PATH)

    def setUp(self):
        self._clear_outputs()
        TEST_CUBE.to_zarr(INPUT_CUBE_PATH)

    def tearDown(self):
        self._clear_outputs()

    def test_defaults(self):
        result = self.invoke_cli(['optimize', INPUT_CUBE_PATH])
        self.assertEqual(0, result.exit_code)

        expected_files = set(INPUT_CUBE_FILE_SET)
        expected_files.add('.zmetadata')
        self.assertEqual(expected_files, list_file_set(OUTPUT_CUBE_OPTIMIZED_DEFAULT_PATH))

    def test_user_output(self):
        result = self.invoke_cli(['optimize', '-o', OUTPUT_CUBE_OPTIMIZED_USER_PATH, '-C', INPUT_CUBE_PATH])
        self.assertEqual(0, result.exit_code)

        expected_files = set(INPUT_CUBE_FILE_SET)
        expected_files.add('.zmetadata')
        expected_files.remove('time/1')
        expected_files.remove('time/2')
        expected_files.remove('time_bnds/1.0')
        expected_files.remove('time_bnds/2.0')
        self.assertEqual(expected_files, list_file_set(OUTPUT_CUBE_OPTIMIZED_USER_PATH))

    def test_in_place(self):
        result = self.invoke_cli(['optimize', '-IC', INPUT_CUBE_PATH])
        self.assertEqual(0, result.exit_code)

        expected_files = set(INPUT_CUBE_FILE_SET)
        expected_files.add('.zmetadata')
        expected_files.remove('time/1')
        expected_files.remove('time/2')
        expected_files.remove('time_bnds/1.0')
        expected_files.remove('time_bnds/2.0')
        self.assertEqual(expected_files, list_file_set(INPUT_CUBE_PATH))


class OptimizeTest(CliDataTest):

    def test_help_option(self):
        result = self.invoke_cli(['optimize', '--help'])
        self.assertEqual(0, result.exit_code)


# test/cli/test_prune.py

import numpy as np
import os.path
import sys

import xarray as xr

from test.cli.helpers import CliTest
from xcube.cli.prune import _delete_block_file
from xcube.core.dsio import rimraf
from xcube.core.dsio import write_cube
from xcube.core.new import new_cube
from xcube.core.verify import assert_cube


class PruneTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['prune', '--help'])
        self.assertEqual(0, result.exit_code)


class PruneDataTest(CliTest):
    TEST_CUBE = "test.zarr"

    def setUp(self) -> None:
        rimraf(self.TEST_CUBE)
        cube = new_cube(time_periods=3,
                        variables=dict(precipitation=np.nan,
                                       temperature=np.nan)) \
            .chunk(dict(time=1, lat=90, lon=90))
        fv_encoding = dict(
            _FillValue=None
        )
        encoding = dict(
            precipitation=fv_encoding,
            temperature=fv_encoding
        )
        cube.to_zarr(self.TEST_CUBE, encoding=encoding)

    def tearDown(self) -> None:
        rimraf(self.TEST_CUBE)

    def test_dry_run(self):
        result = self.invoke_cli(['prune', self.TEST_CUBE, "-vv", "--dry-run"])
        self.assertEqual(0, result.exit_code)
        self.assertEqual(
            (
                "Opening dataset from 'test.zarr'...\n"
                "Identifying empty chunks...\n"
                "Found empty chunks in variable 'precipitation', "
                "deleting block files...\n"
                "Deleted 24 block file(s) for variable 'precipitation'.\n"
                "Found empty chunks in variable 'temperature', "
                "deleting block files...\n"
                "Deleted 24 block file(s) for variable 'temperature'.\n"
                "Done, 48 block file(s) deleted total.\n"
            ),
            result.stdout)
        expected_file_names = sorted([
            '.zarray',
            '.zattrs',
            '0.0.0', '0.0.1', '0.0.2', '0.0.3',
            '0.1.0', '0.1.1', '0.1.2', '0.1.3',
            '1.0.0', '1.0.1', '1.0.2', '1.0.3',
            '1.1.0', '1.1.1', '1.1.2', '1.1.3',
            '2.0.0', '2.0.1', '2.0.2', '2.0.3',
            '2.1.0', '2.1.1', '2.1.2', '2.1.3'
        ])
        self.assertEqual(expected_file_names,
                         sorted(os.listdir('test.zarr/precipitation')))
        self.assertEqual(expected_file_names,
                         sorted(os.listdir('test.zarr/temperature')))
        ds = xr.open_zarr('test.zarr')
        assert_cube(ds)
        self.assertIn('precipitation', ds)
        self.assertEqual((3, 180, 360), ds.precipitation.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.precipitation.dims)
        self.assertIn('temperature', ds)
        self.assertEqual((3, 180, 360), ds.temperature.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.temperature.dims)

    def test_no_dry_run(self):
        result = self.invoke_cli(['prune', self.TEST_CUBE, "-vv"])
        self.assertEqual(0, result.exit_code)
        self.assertEqual(
            (
                "Opening dataset from 'test.zarr'...\n"
                "Identifying empty chunks...\n"
                "Found empty chunks in variable 'precipitation', "
                "deleting block files...\n"
                "Deleted 24 block file(s) for variable 'precipitation'.\n"
                "Found empty chunks in variable 'temperature', "
                "deleting block files...\n"
                "Deleted 24 block file(s) for variable 'temperature'.\n"
                "Done, 48 block file(s) deleted total.\n"
            ),
            result.stdout)
        expected_file_names = sorted(['.zarray', '.zattrs'])
        self.assertEqual(expected_file_names,
                         sorted(os.listdir('test.zarr/precipitation')))
        self.assertEqual(expected_file_names,
                         sorted(os.listdir('test.zarr/temperature')))
        ds = xr.open_zarr('test.zarr')
        assert_cube(ds)
        self.assertIn('precipitation', ds)
        self.assertEqual((3, 180, 360), ds.precipitation.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.precipitation.dims)
        self.assertIn('temperature', ds)
        self.assertEqual((3, 180, 360), ds.temperature.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.temperature.dims)

    def test_delete_block_file(self):
        actual_message = None

        def monitor(message, level):
            nonlocal actual_message
            actual_message = message

        actual_message = None
        ok = _delete_block_file(self.TEST_CUBE,
                                'precipitation', (0, 3, 76),
                                True, monitor=monitor)
        self.assertFalse(ok)
        file_path = os.path.join(self.TEST_CUBE,
                                 'precipitation', '0.3.76')
        dir_path = os.path.join(self.TEST_CUBE,
                                'precipitation', '0', '3', '76')
        self.assertEqual(f"Error: could find neither block file "
                         f"{file_path} nor "
                         f"{dir_path}", actual_message)

        block_file = os.path.join(self.TEST_CUBE,
                                  'precipitation', '1.1.0')

        actual_message = None
        ok = _delete_block_file(self.TEST_CUBE,
                                'precipitation', (1, 1, 0),
                                True, monitor=monitor)
        self.assertTrue(ok)
        self.assertEqual(f'Deleting chunk file {block_file}',
                         actual_message)

        actual_message = None
        ok = _delete_block_file(self.TEST_CUBE,
                                'precipitation', (1, 1, 0),
                                False, monitor=monitor)
        self.assertTrue(ok)
        self.assertEqual(f'Deleting chunk file {block_file}',
                         actual_message)
        self.assertFalse(os.path.exists(block_file))

        if sys.platform == 'win32':
            block_file = os.path.join(self.TEST_CUBE,
                                      'precipitation', '1.1.1')
            # Open block, so we cannot delete (Windows only)
            # noinspection PyUnusedLocal
            with open(block_file, 'wb') as fp:
                actual_message = None
                ok = _delete_block_file(self.TEST_CUBE,
                                        'precipitation', (1, 1, 1),
                                        False, monitor=monitor)
                self.assertFalse(ok)
                self.assertIsNotNone(actual_message)
                # noinspection PyUnresolvedReferences
                self.assertTrue(actual_message.startswith(
                    f'Error: failed to delete block file {block_file}: '
                ))


# test/cli/test_rectify.py

import os
from typing import List
import xarray as xr
from test.cli.helpers import CliDataTest, TEST_ZARR_DIR
from xcube.core.verify import assert_cube


class RectifyTest(CliDataTest):

    def outputs(self) -> List[str]:
        return ['out.zarr']

    def test_rectify_without_vars(self):
        """Test that rectify selects all variables when --var is not given."""

        # For now, specify the image geometry explicitly with --size, --point,
        # and --res to avoid triggering an "invalid y_min" ValueError when
        # ImageGeom tries to determine it automatically. Once Issue #303 has
        # been fixed, these options can be omitted.

        result = self.invoke_cli(['rectify',
                                  '--size', '508,253',
                                  '--point', '-179.5,-89.5',
                                  '--res', '0.7071067811865475',
                                  '--crs', 'EPSG:4326',
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('Opening dataset from \'test.zarr\'...\n'
                         'Rectifying...\n'
                         'Writing rectified dataset to \'out.zarr\'...\n'
                         'Done.\n',
                         result.stdout)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('precipitation', ds)
        self.assertIn('temperature', ds)
        self.assertIn('soil_moisture', ds)

    def test_rectify_multiple_comma_separated_vars(self):
        """Test that rectify selects the desired variables when
        multiple --var options, some with multiple comma-separated
        variable names as an argument, are passed."""

        # For now, specify the image geometry explicitly with --size, --point,
        # and --res to avoid triggering an "invalid y_min" ValueError when
        # ImageGeom tries to determine it automatically. Once Issue #303 has
        # been fixed, these options can be omitted.

        result = self.invoke_cli(['rectify',
                                  '--size', '508,253',
                                  '--point', '-179.5,-89.5',
                                  '--res', '0.7071067811865475',
                                  '--crs', 'EPSG:4326',
                                  '--var', 'precipitation,temperature',
                                  '--var', 'soil_moisture',
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('Opening dataset from \'test.zarr\'...\n'
                         'Rectifying...\n'
                         'Writing rectified dataset to \'out.zarr\'...\n'
                         'Done.\n',
                         result.stdout)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('precipitation', ds)
        self.assertIn('temperature', ds)
        self.assertIn('soil_moisture', ds)


# test/cli/test_resample.py

import os.path
from typing import List

import xarray as xr

from test.cli.helpers import CliTest, CliDataTest, TEST_ZARR_DIR
from xcube.core.verify import assert_cube


class ResampleTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['resample', '--help'])
        self.assertEqual(0, result.exit_code)


class ResampleDataTest(CliDataTest):
    def outputs(self) -> List[str]:
        return ['out.zarr', 'resampled.zarr']

    def test_all_defaults(self):
        result = self.invoke_cli(['resample', TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertEqual("Opening cube from 'test.zarr'...\n"
                         "Resampling...\n"
                         "Writing resampled cube to 'out.zarr'...\n"
                         "Done.\n",
                         result.stdout)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('precipitation_mean', ds)
        self.assertIn('temperature_mean', ds)
        self.assertIn('soil_moisture_mean', ds)

    def test_with_output(self):
        result = self.invoke_cli(['resample', TEST_ZARR_DIR, '--output', 'resampled.zarr'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir('resampled.zarr'))
        ds = xr.open_zarr('resampled.zarr')
        assert_cube(ds)
        self.assertIn('precipitation_mean', ds)
        self.assertIn('temperature_mean', ds)
        self.assertIn('soil_moisture_mean', ds)

    def test_with_vars(self):
        result = self.invoke_cli(['resample', TEST_ZARR_DIR, '--vars', 'temperature,precipitation'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('precipitation_mean', ds)
        self.assertIn('temperature_mean', ds)
        self.assertNotIn('soil_moisture_mean', ds)

    def test_downsample_with_multiple_methods(self):
        result = self.invoke_cli(['resample',
                                  '--variables', 'temperature',
                                  '-F', 'all',
                                  '-M', 'mean',
                                  '-M', 'count',
                                  '-M', 'prod',
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('temperature_mean', ds)
        self.assertIn('temperature_count', ds)
        self.assertIn('temperature_prod', ds)

    def test_upsample_with_multiple_methods(self):
        result = self.invoke_cli(['resample',
                                  '--variables', 'temperature',
                                  '-F', '12H',
                                  '-T', '6H',
                                  # '-K', 'quadratic',
                                  # '-M', 'interpolate',
                                  '-M', 'nearest',
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        # self.assertIn('temperature_interpolate', ds)
        self.assertIn('temperature_nearest', ds)


# test/cli/test_serve.py

import os
import os.path
from typing import Mapping

from test.cli.helpers import CliDataTest
from xcube.cli.serve import BASE_ENV_VAR
from xcube.cli.serve import CONFIG_ENV_VAR
from xcube.cli.serve import VIEWER_ENV_VAR
from xcube.cli.serve import main


class ServerCliTest(CliDataTest):
    def test_config(self):
        result = self.invoke_cli(["serve", "--config", "x.yml", "pippo.zarr"])
        self.assertEqual('Error: CONFIG and CUBES cannot be used at the same time.\n',
                         result.output[-58:])
        self.assertEqual(1, result.exit_code)

        with ChangeEnv({CONFIG_ENV_VAR: "x.yml"}):
            result = self.invoke_cli(["serve"])
            self.assertEqual('Error: Configuration file not found: x.yml\n',
                             result.output[-44:])
            self.assertEqual(1, result.exit_code)

    def test_base_dir(self):
        config_path = os.path.join(os.path.dirname(__file__),
                                   '..', '..', 'examples', 'serve', 'demo', 'config.yml')

        with ChangeEnv({BASE_ENV_VAR: None}):
            result = self.invoke_cli(["serve", "--config", config_path, "--base-dir", "pippo/gnarz"])
            self.assertEqual('Error: Base directory not found: pippo/gnarz\n',
                             result.output[-45:])
            self.assertEqual(1, result.exit_code)

        with ChangeEnv({BASE_ENV_VAR: 'pippo/configs'}):
            result = self.invoke_cli(["serve", "--config", config_path])
            self.assertEqual('Error: Base directory not found: pippo/configs\n',
                             result.output[-47:])
            self.assertEqual(1, result.exit_code)

    def test_show(self):
        config_path = os.path.join(os.path.dirname(__file__),
                                   '..', '..', 'examples', 'serve', 'demo', 'config.yml')

        with ChangeEnv({VIEWER_ENV_VAR: None}):
            result = self.invoke_cli(["serve", "--show", "--config", config_path])
            self.assertEqual(f'Error: Option "--show": In order to run the viewer, '
                             f'set environment variable {VIEWER_ENV_VAR} to a valid '
                             f'xcube-viewer deployment or build directory.\n',
                             result.output[-150:])
            self.assertEqual(2, result.exit_code)

        with ChangeEnv({VIEWER_ENV_VAR: "pippo/viewer"}):
            result = self.invoke_cli(["serve", "--show", "--config", config_path])
            self.assertEqual(f'Error: Option "--show": Viewer path set by environment '
                             f'variable {VIEWER_ENV_VAR} '
                             f'must be a directory: pippo/viewer\n',
                             result.output[-116:])
            self.assertEqual(2, result.exit_code)

    def test_help(self):
        with self.assertRaises(SystemExit):
            main(args=["--help"])


class ChangeEnv:
    def __init__(self, env=None, **env_kwargs):
        self._new_env = dict(env) if env else dict()
        self._new_env.update(**env_kwargs)
        self._old_env = None

    def __enter__(self) -> Mapping[str, str]:
        self._old_env = dict()
        for k, v in self._new_env.items():
            self._old_env[k] = os.environ.get(k)
            if v is not None:
                os.environ[k] = str(v)
            elif k in os.environ:
                del os.environ[k]
        return os.environ

    def __exit__(self, *error):
        for k, v in self._old_env.items():
            if v is not None:
                os.environ[k] = v
            elif k in os.environ:
                del os.environ[k]


# test/cli/test_tile.py

import os.path
from typing import List, Tuple

from test.cli.helpers import CliDataTest
from test.cli.helpers import CliTest
from test.cli.helpers import TEST_NC_FILE, TEST_ZARR_DIR


class TileTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['tile', '--help'])
        self.assertEqual(0, result.exit_code)


class LevelDataTest(CliDataTest):
    TEST_OUTPUT = "out.tiles"

    def outputs(self) -> List[str]:
        return [LevelDataTest.TEST_OUTPUT, 'my.tiles', 'config.yml']

    def time_periods(self) -> int:
        return 2

    def test_all_defaults(self):
        result = self.invoke_cli(['tile', TEST_ZARR_DIR])
        self._assert_result_ok(result, LevelDataTest.TEST_OUTPUT, [
            'out.tiles/precipitation/0/0/0/0.png',
            'out.tiles/precipitation/0/0/0/1.png',
            'out.tiles/precipitation/0/tilemapresource.xml',
            'out.tiles/precipitation/1/0/0/0.png',
            'out.tiles/precipitation/1/0/0/1.png',
            'out.tiles/precipitation/1/tilemapresource.xml',
            'out.tiles/precipitation/metadata.json',
            'out.tiles/soil_moisture/0/0/0/0.png',
            'out.tiles/soil_moisture/0/0/0/1.png',
            'out.tiles/soil_moisture/0/tilemapresource.xml',
            'out.tiles/soil_moisture/1/0/0/0.png',
            'out.tiles/soil_moisture/1/0/0/1.png',
            'out.tiles/soil_moisture/1/tilemapresource.xml',
            'out.tiles/soil_moisture/metadata.json',
            'out.tiles/temperature/0/0/0/0.png',
            'out.tiles/temperature/0/0/0/1.png',
            'out.tiles/temperature/0/tilemapresource.xml',
            'out.tiles/temperature/1/0/0/0.png',
            'out.tiles/temperature/1/0/0/1.png',
            'out.tiles/temperature/1/tilemapresource.xml',
            'out.tiles/temperature/metadata.json'
        ])

    def test_with_output(self):
        result = self.invoke_cli(['tile', TEST_ZARR_DIR, '--output', 'my.tiles', '-vvv'])
        self._assert_result_ok(result, 'my.tiles', [
            'my.tiles/precipitation/0/0/0/0.png',
            'my.tiles/precipitation/0/0/0/1.png',
            'my.tiles/precipitation/0/tilemapresource.xml',
            'my.tiles/precipitation/1/0/0/0.png',
            'my.tiles/precipitation/1/0/0/1.png',
            'my.tiles/precipitation/1/tilemapresource.xml',
            'my.tiles/precipitation/metadata.json',
            'my.tiles/soil_moisture/0/0/0/0.png',
            'my.tiles/soil_moisture/0/0/0/1.png',
            'my.tiles/soil_moisture/0/tilemapresource.xml',
            'my.tiles/soil_moisture/1/0/0/0.png',
            'my.tiles/soil_moisture/1/0/0/1.png',
            'my.tiles/soil_moisture/1/tilemapresource.xml',
            'my.tiles/soil_moisture/metadata.json',
            'my.tiles/temperature/0/0/0/0.png',
            'my.tiles/temperature/0/0/0/1.png',
            'my.tiles/temperature/0/tilemapresource.xml',
            'my.tiles/temperature/1/0/0/0.png',
            'my.tiles/temperature/1/0/0/1.png',
            'my.tiles/temperature/1/tilemapresource.xml',
            'my.tiles/temperature/metadata.json'
        ])

    def test_with_tile_size(self):
        result = self.invoke_cli(['tile', TEST_ZARR_DIR, '-t', '90'])
        self._assert_result_ok(result, LevelDataTest.TEST_OUTPUT, [
            'out.tiles/precipitation/0/0/0/0.png',
            'out.tiles/precipitation/0/0/1/0.png',
            'out.tiles/precipitation/0/1/0/0.png',
            'out.tiles/precipitation/0/1/0/1.png',
            'out.tiles/precipitation/0/1/1/0.png',
            'out.tiles/precipitation/0/1/1/1.png',
            'out.tiles/precipitation/0/1/2/0.png',
            'out.tiles/precipitation/0/1/2/1.png',
            'out.tiles/precipitation/0/1/3/0.png',
            'out.tiles/precipitation/0/1/3/1.png',
            'out.tiles/precipitation/0/tilemapresource.xml',
            'out.tiles/precipitation/1/0/0/0.png',
            'out.tiles/precipitation/1/0/1/0.png',
            'out.tiles/precipitation/1/1/0/0.png',
            'out.tiles/precipitation/1/1/0/1.png',
            'out.tiles/precipitation/1/1/1/0.png',
            'out.tiles/precipitation/1/1/1/1.png',
            'out.tiles/precipitation/1/1/2/0.png',
            'out.tiles/precipitation/1/1/2/1.png',
            'out.tiles/precipitation/1/1/3/0.png',
            'out.tiles/precipitation/1/1/3/1.png',
            'out.tiles/precipitation/1/tilemapresource.xml',
            'out.tiles/precipitation/metadata.json',
            'out.tiles/soil_moisture/0/0/0/0.png',
            'out.tiles/soil_moisture/0/0/1/0.png',
            'out.tiles/soil_moisture/0/1/0/0.png',
            'out.tiles/soil_moisture/0/1/0/1.png',
            'out.tiles/soil_moisture/0/1/1/0.png',
            'out.tiles/soil_moisture/0/1/1/1.png',
            'out.tiles/soil_moisture/0/1/2/0.png',
            'out.tiles/soil_moisture/0/1/2/1.png',
            'out.tiles/soil_moisture/0/1/3/0.png',
            'out.tiles/soil_moisture/0/1/3/1.png',
            'out.tiles/soil_moisture/0/tilemapresource.xml',
            'out.tiles/soil_moisture/1/0/0/0.png',
            'out.tiles/soil_moisture/1/0/1/0.png',
            'out.tiles/soil_moisture/1/1/0/0.png',
            'out.tiles/soil_moisture/1/1/0/1.png',
            'out.tiles/soil_moisture/1/1/1/0.png',
            'out.tiles/soil_moisture/1/1/1/1.png',
            'out.tiles/soil_moisture/1/1/2/0.png',
            'out.tiles/soil_moisture/1/1/2/1.png',
            'out.tiles/soil_moisture/1/1/3/0.png',
            'out.tiles/soil_moisture/1/1/3/1.png',
            'out.tiles/soil_moisture/1/tilemapresource.xml',
            'out.tiles/soil_moisture/metadata.json',
            'out.tiles/temperature/0/0/0/0.png',
            'out.tiles/temperature/0/0/1/0.png',
            'out.tiles/temperature/0/1/0/0.png',
            'out.tiles/temperature/0/1/0/1.png',
            'out.tiles/temperature/0/1/1/0.png',
            'out.tiles/temperature/0/1/1/1.png',
            'out.tiles/temperature/0/1/2/0.png',
            'out.tiles/temperature/0/1/2/1.png',
            'out.tiles/temperature/0/1/3/0.png',
            'out.tiles/temperature/0/1/3/1.png',
            'out.tiles/temperature/0/tilemapresource.xml',
            'out.tiles/temperature/1/0/0/0.png',
            'out.tiles/temperature/1/0/1/0.png',
            'out.tiles/temperature/1/1/0/0.png',
            'out.tiles/temperature/1/1/0/1.png',
            'out.tiles/temperature/1/1/1/0.png',
            'out.tiles/temperature/1/1/1/1.png',
            'out.tiles/temperature/1/1/2/0.png',
            'out.tiles/temperature/1/1/2/1.png',
            'out.tiles/temperature/1/1/3/0.png',
            'out.tiles/temperature/1/1/3/1.png',
            'out.tiles/temperature/1/tilemapresource.xml',
            'out.tiles/temperature/metadata.json'
        ])

    def _assert_result_ok(self, result, output_path: str, actual_file_listing: List[Tuple]):
        self.assertEqual(0, result.exit_code)
        # self.assertRegex(result.stdout, message_regex)
        self.assertTrue(os.path.isdir(output_path))

        expected_file_listing = []
        for root, dirs, files in os.walk(output_path, topdown=False):
            for name in files:
                expected_file_listing.append(os.path.join(root, name).replace('\\', '/'))
        expected_file_listing.sort()

        self.assertEqual(expected_file_listing, actual_file_listing)

    def test_invalid_inputs(self):
        result = self.invoke_cli(["tile",
                                  "-t", "a45",
                                  TEST_ZARR_DIR,
                                  ])
        self._assert_result_not_ok(result,
                                   "Error\\: Invalid tile sizes in TILE_SIZE found: "
                                   "invalid literal for int\\(\\) with base 10\\: 'a45'\n")

        result = self.invoke_cli(["tile",
                                  "-t", "-3",
                                  TEST_NC_FILE,
                                  ])
        self._assert_result_not_ok(result,
                                   "Error\\: Invalid tile sizes in TILE_SIZE found\\: "
                                   "all items must be positive integer numbers\n")

        result = self.invoke_cli(["tile",
                                  "-t", "45,45,45",
                                  TEST_NC_FILE])
        self._assert_result_not_ok(result,
                                   "Error\\: TILE_SIZE must have 2 tile sizes separated by ','\n")

    def _assert_result_not_ok(self, result, message_regex: str):
        self.assertEqual(1, result.exit_code)
        self.assertRegex(result.stdout, message_regex)


# test/cli/test_vars2dim.py

import os

import xarray as xr

from test.cli.helpers import CliDataTest, TEST_ZARR_DIR


class Vars2DimTest(CliDataTest):
    TEST_OUTPUT = "test-vars2dim.zarr"

    def outputs(self):
        return [self.TEST_OUTPUT]

    def test_vars2dim(self):
        result = self.invoke_cli(["vars2dim", TEST_ZARR_DIR])

        output_path = self.TEST_OUTPUT
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(output_path))

        ds = xr.open_zarr(output_path)
        self.assertIn("var", ds.dims)
        self.assertEqual(3, ds.dims["var"])
        self.assertIn("var", ds.coords)
        self.assertIn("data", ds.data_vars)
        var_names = ds["var"]
        self.assertEqual(("var",), var_names.dims)
        self.assertTrue(hasattr(var_names, "encoding"))
        self.assertEqual(3, len(var_names))
        self.assertIn("precipitation", str(var_names[0]))
        self.assertIn("soil_moisture", str(var_names[1]))
        self.assertIn("temperature", str(var_names[2]))


# test/cli/test_verify.py

import numpy as np
import xarray as xr

from test.cli.helpers import CliTest
from xcube.core.dsio import rimraf
from xcube.core.dsio import write_cube
from xcube.core.new import new_cube


class VerifyTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['verify', '--help'])
        self.assertEqual(0, result.exit_code)


class VerifyDataTest(CliTest):
    TEST_CUBE = "test.zarr"

    def setUp(self) -> None:
        rimraf(self.TEST_CUBE)

    def tearDown(self) -> None:
        rimraf(self.TEST_CUBE)

    def test_verify_ok(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
        result = self.invoke_cli(['verify', self.TEST_CUBE])
        self.assertEqual(0, result.exit_code)
        self.assertEqual("Opening cube from 'test.zarr'...\n"
                         "INPUT is a valid cube.\n",
                         result.stdout)

    def test_verify_failure(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        cube["chl"] = xr.DataArray(np.random.rand(cube.dims["lat"], cube.dims["lon"]),
                                   dims=("lat", "lon"),
                                   coords=dict(lat=cube.lat, lon=cube.lon))
        write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)

        result = self.invoke_cli(['verify', self.TEST_CUBE])
        self.assertEqual(3, result.exit_code)
        self.assertEqual("Opening cube from 'test.zarr'...\n"
                         "INPUT is not a valid cube due to the following reasons:\n"
                         "- dimensions of data variable 'chl' must be ('time', ..., 'lat', 'lon'),"
                         " but were ('lat', 'lon') for 'chl'\n"
                         "- dimensions of all data variables must be same, but found ('lat', 'lon')"
                         " for 'chl' and ('time', 'lat', 'lon') for 'precipitation'\n"
                         "- all data variables must have same chunk sizes, but found ((90, 90), (360,))"
                         " for 'chl' and ((3, 2), (90, 90), (180, 180)) for 'precipitation'\n",
                         result.stdout)


# test/cli/test_versions.py

from test.cli.helpers import CliTest
from xcube.util.temp import new_temp_file


class VersionsTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['versions', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_no_args(self):
        result = self.invoke_cli(['versions'])
        self.assertEqual(0, result.exit_code)
        self.assertYaml(result.stdout)

    def test_format(self):
        result = self.invoke_cli(['versions', '-f', 'json'])
        self.assertEqual(0, result.exit_code)
        self.assertJson(result.stdout)

        result = self.invoke_cli(['versions', '-f', 'yaml'])
        self.assertEqual(0, result.exit_code)
        self.assertYaml(result.stdout)

    def test_output(self):
        _, output_path = new_temp_file(suffix='.json')
        result = self.invoke_cli(['versions', '-o', output_path])
        self.assertEqual(0, result.exit_code)
        with open(output_path) as fp:
            text = fp.read()
        self.assertJson(text)

        _, output_path = new_temp_file(suffix='.yml')
        result = self.invoke_cli(['versions', '-o', output_path])
        self.assertEqual(0, result.exit_code)
        with open(output_path) as fp:
            text = fp.read()
        self.assertYaml(text)

    def assertJson(self, json_result: str):
        self.assertIn('{', json_result)
        self.assertIn(':', json_result)
        self.assertIn('}', json_result)

    def assertYaml(self, json_result: str):
        self.assertNotIn('{', json_result)
        self.assertIn(':', json_result)
        self.assertNotIn('}', json_result)


# test/core/diagnosticstore.py

import time
from collections.abc import MutableMapping
from types import MethodType
from typing import TypeVar, Iterator, List, Callable, Any, Tuple

_KT = TypeVar('_KT')
_VT = TypeVar('_VT')
_T_co = TypeVar('_T_co')
_VT_co = TypeVar('_VT_co')


class DiagnosticStore(MutableMapping):

    def __init__(self,
                 delegate: MutableMapping,
                 observer: Callable[[int, float, str, List[Tuple[str, Any]]], None] = None):
        self._delegate = delegate
        self._observer = observer or logging_observer()
        self._counter = 0
        self._add_optional_method('listdir', ['path'])
        self._add_optional_method('rmdir', ['path'])
        self._add_optional_method('rename', ['from_path', 'to_path'])

    def _add_optional_method(self, method_name: str, arg_names: List[str]):
        if hasattr(self._delegate, method_name):
            def method(_self, *args) -> List[str]:
                return _self.call_and_notify(method_name, *[(arg_names[i], args[i]) for i in range(len(args))])

            setattr(self, method_name, MethodType(method, self))

    def call_and_notify(self, method_name: str, *args):
        method = getattr(self._delegate, method_name)

        t0 = time.perf_counter()
        result = method(*(arg[1] for arg in args))
        t1 = time.perf_counter()

        self._counter += 1
        self._observer(self._counter, t1 - t0, method_name, *args)

        return result

    def __contains__(self, k: _KT) -> bool:
        return self.call_and_notify('__contains__', ('k', k))

    def __setitem__(self, k: _KT, v: _VT) -> None:
        return self.call_and_notify('__setitem__', ('k', k), ('v', v))

    def __delitem__(self, k: _KT) -> None:
        return self.call_and_notify('__delitem__', ('k', k))

    def __getitem__(self, k: _KT) -> _VT_co:
        return self.call_and_notify('__getitem__', ('k', k))

    def __len__(self) -> int:
        return self.call_and_notify('__len__')

    def __iter__(self) -> Iterator[_T_co]:
        return self.call_and_notify('__iter__')


def logging_observer(logger_name=None, log_path=None, log_all=False):
    import logging

    observer_logger = logging.getLogger(logger_name or 'diagnosticstore')

    if log_all:
        logger = logging.getLogger()
    else:
        logger = observer_logger

    logger.setLevel(logging.DEBUG)
    handler = logging.FileHandler(log_path or 'diagnosticstore.log')
    handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(name)s: %(message)s'))
    logger.addHandler(handler)

    def observer(counter, time_needed, method_name, *args):
        msg = f'call #{counter}: {method_name}('
        msg += ', '.join(map(lambda x: f'{x[0]}={repr(x[1])}', args))
        msg += f'), took {int(1000 * time_needed)} ms'
        observer_logger.info(msg)

    return observer


# test/core/test_ancvar.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.ancvar import find_ancillary_var_names


class FindAncillaryVarNamesTest(unittest.TestCase):
    @classmethod
    def _new_array(self, dims=None, **attrs):
        dims = dims or [('time', 3), ('lat', 3), ('lon', 3)]
        s = 1
        for d in dims:
            s *= d[1]
        return xr.DataArray(np.linspace(0, 1, s).reshape([s for _, s in dims]), dims=([n for n, _ in dims]),
                            attrs=attrs)

    def test_strict_cf_convention(self):
        a = self._new_array(standard_name='a', ancillary_variables='a_stdev b c')
        a_stdev = self._new_array(standard_name='a standard_error')
        a_uncert = self._new_array(standard_name='a standard_error')
        a_count = self._new_array(standard_name='a number_of_observations')
        b = self._new_array(standard_name='b')
        b_stdev = self._new_array(standard_name='b standard_error')
        c = self._new_array(standard_name='c')

        ds = xr.Dataset(dict(a=a))
        self.assertEqual({}, find_ancillary_var_names(ds, 'a'))

        ds = xr.Dataset(dict(a=a, a_stdev=a_stdev, a_uncert=a_uncert, a_count=a_count, b=b, b_stdev=b_stdev, c=c))
        self.assertEqual({'': {'c', 'b'}, 'standard_error': {'a_stdev'}},
                         find_ancillary_var_names(ds, 'a'))

    def test_less_strict_cf_convention(self):
        a = self._new_array(standard_name='a')
        a_stdev = self._new_array(standard_name='a standard_error')
        a_uncert = self._new_array(standard_name='a standard_error')
        a_count = self._new_array(standard_name='a number_of_observations')
        b = self._new_array(standard_name='b')
        b_stdev = self._new_array(standard_name='b standard_error')

        ds = xr.Dataset(dict(a=a))
        self.assertEqual({}, find_ancillary_var_names(ds, 'a'))

        ds = xr.Dataset(dict(a=a, a_stdev=a_stdev, a_uncert=a_uncert, a_count=a_count, b=b, b_stdev=b_stdev))
        self.assertEqual({'number_of_observations': {'a_count'},
                          'standard_error': {'a_stdev', 'a_uncert'}},
                         find_ancillary_var_names(ds, 'a'))

    def test_xcube_prefix_convention(self):
        a = self._new_array()
        a_std = self._new_array()
        a_count = self._new_array()
        b = self._new_array()
        b_std = self._new_array()

        ds = xr.Dataset(dict(a=a))
        self.assertEqual({}, find_ancillary_var_names(ds, 'a'))

        ds = xr.Dataset(dict(a=a, a_std=a_std, a_count=a_count, b=b, b_std=b_std))
        self.assertEqual({'number_of_observations': {'a_count'},
                          'standard_error': {'a_std'}},
                         find_ancillary_var_names(ds, 'a'))


# test/core/test_chunk.py

import collections
import unittest

import numpy as np

from test.sampledata import new_test_dataset
from xcube.core.chunk import chunk_dataset
from xcube.core.chunk import compute_chunk_slices
from xcube.core.chunk import get_empty_dataset_chunks


class ChunkDatasetTest(unittest.TestCase):

    def test_chunk_dataset(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02",
                                    "2010-01-03", "2010-01-04",
                                    "2010-01-05"],
                                   precipitation=0.4, temperature=275.2)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=dict(time=1,
                                                         lat=10, lon=20),
                                        format_name="zarr")
        self.assertEqual({'chunks': (1, 10, 20)},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({'chunks': (1, 10, 20)},
                         chunked_dataset.temperature.encoding)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=dict(time=1,
                                                         lat=20, lon=40),
                                        format_name="netcdf4")
        self.assertEqual({'chunksizes': (1, 20, 40)},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({'chunksizes': (1, 20, 40)},
                         chunked_dataset.temperature.encoding)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=dict(time=1,
                                                         lat=20, lon=40))
        self.assertEqual({}, chunked_dataset.precipitation.encoding)
        self.assertEqual({}, chunked_dataset.temperature.encoding)

        dataset = dataset.chunk(dict(time=2, lat=10, lon=20))

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=None,
                                        format_name="zarr")
        self.assertEqual({}, chunked_dataset.precipitation.encoding)
        self.assertEqual({}, chunked_dataset.temperature.encoding)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes={},
                                        format_name="zarr")
        self.assertEqual({'chunks': (2, 10, 20)},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({'chunks': (2, 10, 20)},
                         chunked_dataset.temperature.encoding)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=dict(time=1),
                                        format_name="zarr")
        self.assertEqual({'chunks': (1, 10, 20)},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({'chunks': (1, 10, 20)},
                         chunked_dataset.temperature.encoding)

    def test_unchunk_dataset(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02",
                                    "2010-01-03", "2010-01-04",
                                    "2010-01-05"],
                                   precipitation=0.4,
                                   temperature=275.2)

        for var in dataset.data_vars.values():
            var.encoding.update({"chunks": (5, 180, 360),
                                 "_FillValue": -999.0})

        chunked_dataset = chunk_dataset(dataset, format_name="zarr")
        self.assertEqual({"_FillValue": -999.0},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({"_FillValue": -999.0},
                         chunked_dataset.temperature.encoding)


class GetEmptyDatasetChunksTest(unittest.TestCase):

    def test_not_chunked(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02"],
                                   precipitation=0.4,
                                   temperature=275.2)
        empty_dataset_chunks = get_empty_dataset_chunks(dataset)
        self.assertIsInstance(empty_dataset_chunks, collections.Iterator)
        self.assertFalse(isinstance(empty_dataset_chunks, (list, tuple)))
        self.assertEqual([('precipitation', ()),
                          ('temperature', ())],
                         [(v, tuple(c)) for v, c in empty_dataset_chunks])

    def test_non_empty(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02"],
                                   precipitation=0.4,
                                   temperature=275.2) \
            .chunk(dict(time=1, lat=90, lon=90))
        empty_dataset_chunks = get_empty_dataset_chunks(dataset)
        self.assertIsInstance(empty_dataset_chunks, collections.Iterator)
        self.assertFalse(isinstance(empty_dataset_chunks, (list, tuple)))
        self.assertEqual([('precipitation', ()),
                          ('temperature', ())],
                         [(v, tuple(c)) for v, c in empty_dataset_chunks])

    def test_all_empty(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02"],
                                   precipitation=np.nan,
                                   temperature=np.nan) \
            .chunk(dict(time=1, lat=90, lon=90))
        empty_dataset_chunks = get_empty_dataset_chunks(dataset)
        self.assertIsInstance(empty_dataset_chunks, collections.Iterator)
        self.assertFalse(isinstance(empty_dataset_chunks, (list, tuple)))
        self.assertEqual([('precipitation', ((0, 0, 0),
                                             (0, 0, 1),
                                             (0, 0, 2),
                                             (0, 0, 3),
                                             (0, 1, 0),
                                             (0, 1, 1),
                                             (0, 1, 2),
                                             (0, 1, 3),
                                             (1, 0, 0),
                                             (1, 0, 1),
                                             (1, 0, 2),
                                             (1, 0, 3),
                                             (1, 1, 0),
                                             (1, 1, 1),
                                             (1, 1, 2),
                                             (1, 1, 3))),
                          ('temperature', ((0, 0, 0),
                                           (0, 0, 1),
                                           (0, 0, 2),
                                           (0, 0, 3),
                                           (0, 1, 0),
                                           (0, 1, 1),
                                           (0, 1, 2),
                                           (0, 1, 3),
                                           (1, 0, 0),
                                           (1, 0, 1),
                                           (1, 0, 2),
                                           (1, 0, 3),
                                           (1, 1, 0),
                                           (1, 1, 1),
                                           (1, 1, 2),
                                           (1, 1, 3)))],
                         [(v, tuple(c)) for v, c in empty_dataset_chunks])


class ComputeChunkSlicesTest(unittest.TestCase):

    def test_compute_chunk_slices(self):
        chunk_slices = compute_chunk_slices(((1, 1),
                                             (90, 90), (90, 90, 90, 90)))
        self.assertEqual([((0, 0, 0), ((0, 1), (0, 90), (0, 90))),
                          ((0, 0, 1), ((0, 1), (0, 90), (90, 180))),
                          ((0, 0, 2), ((0, 1), (0, 90), (180, 270))),
                          ((0, 0, 3), ((0, 1), (0, 90), (270, 360))),
                          ((0, 1, 0), ((0, 1), (90, 180), (0, 90))),
                          ((0, 1, 1), ((0, 1), (90, 180), (90, 180))),
                          ((0, 1, 2), ((0, 1), (90, 180), (180, 270))),
                          ((0, 1, 3), ((0, 1), (90, 180), (270, 360))),
                          ((1, 0, 0), ((1, 2), (0, 90), (0, 90))),
                          ((1, 0, 1), ((1, 2), (0, 90), (90, 180))),
                          ((1, 0, 2), ((1, 2), (0, 90), (180, 270))),
                          ((1, 0, 3), ((1, 2), (0, 90), (270, 360))),
                          ((1, 1, 0), ((1, 2), (90, 180), (0, 90))),
                          ((1, 1, 1), ((1, 2), (90, 180), (90, 180))),
                          ((1, 1, 2), ((1, 2), (90, 180), (180, 270))),
                          ((1, 1, 3), ((1, 2), (90, 180), (270, 360)))],
                         list(chunk_slices))


# test/core/test_chunkstore.py

import unittest
import warnings
from typing import Tuple

import numpy as np
import xarray as xr
from zarr.storage import MemoryStore

from xcube.core.chunkstore import ChunkStore
from xcube.core.chunkstore import LoggingStore
from xcube.core.chunkstore import MutableLoggingStore


class ChunkStoreTest(unittest.TestCase):

    def test_chunk_store(self):
        index_var = gen_index_var(dims=('time', 'lat', 'lon'),
                                  shape=(4, 8, 16),
                                  chunks=(2, 4, 8))
        self.assertIsNotNone(index_var)
        self.assertEqual((4, 8, 16), index_var.shape)
        self.assertEqual(((2, 2), (4, 4), (8, 8)), index_var.chunks)
        self.assertEqual(('time', 'lat', 'lon'), index_var.dims)

        visited_indexes = set()

        def index_var_ufunc(index_var_):
            if index_var_.size < 6:
                warnings.warn(f"weird variable of size {index_var_.size} received!")
                return
            nonlocal visited_indexes
            index = tuple(map(int, index_var_.ravel()[0:6]))
            visited_indexes.add(index)
            return index_var_

        result = xr.apply_ufunc(index_var_ufunc, index_var, dask='parallelized', output_dtypes=[index_var.dtype])
        self.assertIsNotNone(result)
        self.assertEqual((4, 8, 16), result.shape)
        self.assertEqual(((2, 2), (4, 4), (8, 8)), result.chunks)
        self.assertEqual(('time', 'lat', 'lon'), result.dims)

        values = result.values
        self.assertEqual((4, 8, 16), values.shape)
        np.testing.assert_array_equal(index_var.values, values)

        print(visited_indexes)

        self.assertEqual(8, len(visited_indexes))
        self.assertEqual({
            (0, 2, 0, 4, 0, 8),
            (2, 4, 0, 4, 0, 8),
            (0, 2, 4, 8, 0, 8),
            (2, 4, 4, 8, 0, 8),
            (0, 2, 0, 4, 8, 16),
            (2, 4, 0, 4, 8, 16),
            (0, 2, 4, 8, 8, 16),
            (2, 4, 4, 8, 8, 16),
        }, visited_indexes)


def gen_index_var(dims, shape, chunks):
    # noinspection PyUnusedLocal
    def get_chunk(cube_store: ChunkStore, name: str, index: Tuple[int, ...]) -> bytes:
        data = np.zeros(cube_store.chunks, dtype=np.uint64)
        data_view = data.ravel()
        if data_view.base is not data:
            raise ValueError('view expected')
        if data_view.size < cube_store.ndim * 2:
            raise ValueError('size too small')
        for i in range(cube_store.ndim):
            j1 = cube_store.chunks[i] * index[i]
            j2 = j1 + cube_store.chunks[i]
            data_view[2 * i] = j1
            data_view[2 * i + 1] = j2
        return data.tobytes()

    store = ChunkStore(dims, shape, chunks)
    store.add_lazy_array('__index_var__', '<u8', get_chunk=get_chunk)

    ds = xr.open_zarr(store)
    return ds.__index_var__


class LoggingStoreTest(unittest.TestCase):
    def test_immutable_new(self):
        self.assertReadOk(LoggingStore.new(self.original_store))

    def test_mutable_new(self):
        self.assertWriteOk(LoggingStore.new(self.original_store))

    def test_immutable(self):
        self.assertReadOk(LoggingStore(self.original_store))

    def test_mutable(self):
        self.assertWriteOk(MutableLoggingStore(self.original_store))

    def setUp(self) -> None:
        self.zattrs_value = bytes()
        self.original_store = MemoryStore()
        self.original_store.update({'chl/.zattrs': self.zattrs_value})

    def assertReadOk(self, logging_store: LoggingStore):
        # noinspection PyUnresolvedReferences
        self.assertEqual(['.zattrs'],
                         logging_store.listdir('chl'))
        # noinspection PyUnresolvedReferences
        self.assertEqual(0,
                         logging_store.getsize('chl'))
        self.assertEqual({'chl/.zattrs'},
                         set(logging_store.keys()))
        self.assertEqual(['chl/.zattrs'],
                         list(iter(logging_store)))
        self.assertTrue('chl/.zattrs' in logging_store)
        self.assertEqual(1,
                         len(logging_store))
        self.assertEqual(self.zattrs_value,
                         logging_store.get('chl/.zattrs'))
        # assert original_store not changed
        self.assertEqual({'chl/.zattrs'},
                         set(self.original_store.keys()))

    def assertWriteOk(self, logging_store: MutableLoggingStore):
        zarray_value = bytes()
        logging_store['chl/.zarray'] = zarray_value
        self.assertEqual({'chl/.zattrs',
                          'chl/.zarray'},
                         set(self.original_store.keys()))
        del logging_store['chl/.zarray']
        self.assertEqual({'chl/.zattrs'},
                         set(self.original_store.keys()))


# test/core/test_compute.py

import os
import unittest
from typing import Any, Dict, Tuple

import numpy as np
import xarray as xr

from xcube.core.chunk import chunk_dataset
from xcube.core.compute import CubeFuncOutput
from xcube.core.compute import compute_cube
from xcube.core.compute import compute_dataset
from xcube.core.new import new_cube
from xcube.core.schema import CubeSchema


class ComputeCubeTest(unittest.TestCase):

    def setUp(self) -> None:
        cube = new_cube(width=360,
                        height=180,
                        time_periods=6,
                        variables=dict(analysed_sst=275.3,
                                       analysis_error=2.1))
        cube = chunk_dataset(cube, dict(time=3, lat=90, lon=90))
        self.cube = cube

    def test_without_inputs(self):
        calls = []

        def my_cube_func(input_params: Dict[str, Any] = None,
                         dim_coords: Dict[str, np.ndarray] = None,
                         dim_ranges: Dict[str, Tuple[int, int]] = None) -> CubeFuncOutput:
            nonlocal calls
            calls.append((input_params, dim_coords, dim_ranges))
            lon_range = dim_ranges['lon']
            lat_range = dim_ranges['lat']
            time_range = dim_ranges['time']
            n_lon = lon_range[1] - lon_range[0]
            n_lat = lat_range[1] - lat_range[0]
            n_time = time_range[1] - time_range[0]
            fill_value = input_params['fill_value']
            return np.full((n_time, n_lat, n_lon), fill_value, dtype=np.float64)

        output_cube = compute_cube(my_cube_func,
                                   input_cube_schema=CubeSchema.new(self.cube),
                                   input_params=dict(fill_value=0.74))

        self.assertIsInstance(output_cube, xr.Dataset)
        self.assertIn('output', output_cube.data_vars)
        output_var = output_cube.output
        self.assertEqual(0, len(calls))
        self.assertEqual(('time', 'lat', 'lon'), output_var.dims)
        self.assertEqual((6, 180, 360), output_var.shape)

        values = output_var.values
        self.assertEqual(2 * 2 * 4, len(calls))
        self.assertEqual((6, 180, 360), values.shape)
        self.assertAlmostEqual(0.74, values[0, 0, 0])
        self.assertAlmostEqual(0.74, values[-1, -1, -1])

    def test_from_two_inputs(self):
        calls = []

        def my_cube_func(analysed_sst: np.ndarray,
                         analysis_error: np.ndarray,
                         input_params: Dict[str, Any] = None,
                         dim_coords: Dict[str, np.ndarray] = None,
                         dim_ranges: Dict[str, Tuple[int, int]] = None) -> CubeFuncOutput:
            nonlocal calls
            calls.append((analysed_sst, analysis_error, input_params, dim_coords, dim_ranges))
            return analysed_sst.data + input_params['factor'] * analysis_error

        output_cube = compute_cube(my_cube_func,
                                   self.cube,
                                   input_var_names=['analysed_sst', 'analysis_error'],
                                   input_params=dict(factor=0.5))

        self.assertIsInstance(output_cube, xr.Dataset)
        self.assertIn('output', output_cube.data_vars)
        output_var = output_cube.output
        self.assertEqual(0, len(calls))
        self.assertEqual(('time', 'lat', 'lon'), output_var.dims)
        self.assertEqual((6, 180, 360), output_var.shape)
        self.assertEqual(((3, 3), (90, 90), (90, 90, 90, 90)), output_var.chunks)

        values = output_var.values
        self.assertEqual(2 * 2 * 4, len(calls))
        self.assertEqual((6, 180, 360), values.shape)
        self.assertAlmostEqual(275.3 + 0.5 * 2.1, values[0, 0, 0])
        self.assertAlmostEqual(275.3 + 0.5 * 2.1, values[-1, -1, -1])

    def test_from_two_inputs_reduce_time(self):
        calls = []

        def my_cube_func(analysed_sst: np.ndarray,
                         analysis_error: np.ndarray) -> CubeFuncOutput:
            nonlocal calls
            calls.append((analysed_sst, analysis_error))
            analysed_sst_mean = np.nanmean(analysed_sst, -1)
            analysis_error_max = np.nanmax(analysis_error, -1)
            return analysed_sst_mean + analysis_error_max

        output_dataset = compute_dataset(my_cube_func,
                                         self.cube,
                                         input_var_names=['analysed_sst', 'analysis_error'],
                                         output_var_name='analysed_sst_max',
                                         output_var_dims={'lat', 'lon'})

        self.assertIsInstance(output_dataset, xr.Dataset)
        self.assertIn('analysed_sst_max', output_dataset.data_vars)
        output_var = output_dataset.analysed_sst_max
        self.assertEqual(0, len(calls))  # call from dask with 0-dim chunks
        self.assertEqual(('lat', 'lon'), output_var.dims)
        self.assertEqual((180, 360), output_var.shape)
        self.assertEqual(((90, 90), (90, 90, 90, 90)), output_var.chunks)

        values = output_var.values
        self.assertEqual(2 * 4, len(calls))
        self.assertEqual((180, 360), values.shape)
        self.assertAlmostEqual(275.3 + 2.1, values[0, 0])
        self.assertAlmostEqual(275.3 + 2.1, values[-1, -1])

    def test_invalid_cube_func(self):
        def my_cube_func(analysis_error: np.ndarray) -> CubeFuncOutput:
            return analysis_error + 1

        with self.assertRaises(ValueError) as cm:
            compute_cube(my_cube_func,
                         self.cube,
                         input_var_names=['analysed_sst', 'analysis_error'],
                         input_params=dict(factor=0.5))
        self.assertEqual("invalid cube_func 'my_cube_func': expected 2 arguments, "
                         "but got analysis_error",
                         f'{cm.exception}')

        def my_cube_func(analysed_sst, dim_coords, analysis_error) -> CubeFuncOutput:
            return analysed_sst + analysis_error

        with self.assertRaises(ValueError) as cm:
            compute_cube(my_cube_func,
                         self.cube,
                         input_var_names=['analysed_sst', 'analysis_error'],
                         input_params=dict(factor=0.5))
        self.assertEqual("invalid cube_func 'my_cube_func': "
                         "any argument must occur before any of input_params, dim_coords, dim_ranges, "
                         "but got analysed_sst, dim_coords, analysis_error",
                         f'{cm.exception}')

        def my_cube_func(dim_ranges, *vars) -> CubeFuncOutput:
            return vars[0] + vars[0]

        with self.assertRaises(ValueError) as cm:
            compute_cube(my_cube_func,
                         self.cube,
                         input_var_names=['analysed_sst', 'analysis_error'],
                         input_params=dict(factor=0.5))
        self.assertEqual("invalid cube_func 'my_cube_func': "
                         "any argument must occur before any of input_params, dim_coords, dim_ranges, "
                         "but got dim_ranges before *vars",
                         f'{cm.exception}')

    def test_inspect(self):
        """This test is about making sure we get expected results from inspect.getfullargspec(func)"""

        def func():
            pass

        self._assert_inspect(func, exp_args=[], exp_kwonlyargs=[], exp_annotations={})

        # noinspection PyUnusedLocal
        def func(*input_vars):
            pass

        self._assert_inspect(func,
                             exp_args=[],
                             exp_varargs='input_vars',
                             exp_kwonlyargs=[],
                             exp_annotations={})

        # noinspection PyUnusedLocal
        def func(*input_vars, input_params, dim_coords):
            pass

        self._assert_inspect(func,
                             exp_args=[],
                             exp_varargs='input_vars',
                             exp_kwonlyargs=['input_params', 'dim_coords'],
                             exp_annotations={})

        # noinspection PyUnusedLocal
        def func(a, b, c, *input_vars, dim_ranges, input_params):
            pass

        self._assert_inspect(func,
                             exp_args=['a', 'b', 'c'],
                             exp_varargs='input_vars',
                             exp_kwonlyargs=['dim_ranges', 'input_params'],
                             exp_annotations={})

        # noinspection PyUnusedLocal
        def func(a, b, c, dim_ranges, input_params):
            pass

        self._assert_inspect(func,
                             exp_args=['a', 'b', 'c', 'dim_ranges', 'input_params'],
                             exp_kwonlyargs=[],
                             exp_annotations={})

    def _assert_inspect(self,
                        func,
                        exp_args=None,
                        exp_varargs=None,
                        exp_varkw=None,
                        exp_defaults=None,
                        exp_kwonlyargs=None,
                        exp_kwonlydefaults=None,
                        exp_annotations=None):
        import inspect
        argspec = inspect.getfullargspec(func)
        self.assertEqual(exp_args, argspec[0], msg='args')
        self.assertEqual(exp_varargs, argspec[1], msg='varargs')
        self.assertEqual(exp_varkw, argspec[2], msg='varkw')
        self.assertEqual(exp_defaults, argspec[3], msg='defaults')
        self.assertEqual(exp_kwonlyargs, argspec[4], msg='kwonlyargs')
        self.assertEqual(exp_kwonlydefaults, argspec[5], msg='kwonlydefaults')
        self.assertEqual(exp_annotations, argspec[6], msg='annotations')

    def test_xarray_apply_with_dim_reduction(self):
        def compute_block(block: np.ndarray, axis: int = None):
            # print('--> block:', block.shape)
            result = np.nanmean(block, axis=axis)
            # print('<-- result:', result.shape)
            return result

        def compute(obj: xr.Dataset, dim: str):
            return xr.apply_ufunc(compute_block, obj,
                                  kwargs=dict(axis=-1),  # note: apply always moves core dimensions to the end
                                  dask='parallelized',
                                  input_core_dims=[[dim]],
                                  output_dtypes=[np.float64])

        ds = xr.open_zarr(os.path.join(os.path.dirname(__file__), '../../examples/serve/demo/cube-1-250-250.zarr'))
        var = ds.conc_chl

        var_rechunked = var.chunk({'time': -1, 'lat': 250, 'lon': 250})
        var_computed = compute(var_rechunked, 'time')

        # This assertion succeeds
        self.assertEqual((1000, 2000), var_computed.shape)

        # Trigger computations of all chunks
        values = var_computed.values
        # This assertion succeeds fails, because values.shape is now (250, 250).
        # This must be an error in xarray or dask.
        self.assertEqual((1000, 2000), values.shape)

    def test_xarray_map_blocks(self):
        def compute_block(block: xr.Dataset):
            # print('--> block:', block)
            return block

        def compute(obj: xr.Dataset, dim: str):
            return obj.map_blocks(compute_block)

        ds = xr.open_zarr(os.path.join(os.path.dirname(__file__), '../../examples/serve/demo/cube-1-250-250.zarr'))
        var = ds.conc_chl

        var_computed = compute(var, 'time')

        # This assertion succeeds
        self.assertEqual((5, 1000, 2000), var_computed.shape)

        # Trigger computations of all chunks
        values = var_computed.values
        # This assertion succeeds fails, because values.shape is now (250, 250).
        # This must be an error in xarray or dask.
        self.assertEqual((5, 1000, 2000), values.shape)


# test/core/test_dsio.py

import os
import os.path
import unittest
from typing import Set

import boto3
import fsspec
import numpy as np
import pandas as pd
import xarray as xr

from test.s3test import MOTO_SERVER_ENDPOINT_URL, S3Test
from test.sampledata import new_test_dataset
from xcube.core.dsio import CsvDatasetIO, DatasetIO, MemDatasetIO, Netcdf4DatasetIO, ZarrDatasetIO, find_dataset_io, \
    query_dataset_io, get_path_or_s3_store, write_cube, split_s3_url, parse_s3_url_and_kwargs, open_cube
from xcube.core.dsio import open_dataset, write_dataset
from xcube.core.new import new_cube

TEST_NC_FILE = "test.nc"
TEST_NC_FILE_2 = "test-2.nc"


@unittest.skip('Execute manually to test with cube on S3, requires S3 credentials')
class OpenDatasetTest(unittest.TestCase):
    CUBE = 'eurodatacube-test/xcube-gen-f8207f0e-636d-4267-8ac5-abbdc5d121fc.zarr'

    def test_open_dataset(self):
        ds = open_dataset(f's3://{self.CUBE}')
        self.assertIn('B02', ds)
        print(ds)

    def test_plain(self):
        import s3fs

        def _open_cube(path):
            return xr.open_zarr(s3fs.S3Map(path, s3=s3fs.S3FileSystem()))

        ds = _open_cube(self.CUBE)
        self.assertIn('B02', ds)
        print(ds)


class OpenWriteDatasetTest(unittest.TestCase):
    def setUp(self):
        super().setUp()
        self.dataset = new_cube(variables=dict(precipitation=0.2, temperature=279.1))
        self.dataset.to_netcdf(TEST_NC_FILE, mode="w")
        self.dataset.close()

    def tearDown(self):
        self.dataset = None
        os.remove(TEST_NC_FILE)
        super().tearDown()

    def test_open_dataset(self):
        with open_dataset(TEST_NC_FILE) as ds:
            self.assertIsNotNone(ds)
            np.testing.assert_array_equal(ds.time.values, self.dataset.time.values)
            np.testing.assert_array_equal(ds.lat.values, self.dataset.lat.values)
            np.testing.assert_array_equal(ds.lon.values, self.dataset.lon.values)

    def test_write_dataset(self):

        dataset = new_cube()
        try:
            write_dataset(dataset, TEST_NC_FILE_2)
            self.assertTrue(os.path.isfile(TEST_NC_FILE_2))
        finally:
            if os.path.isfile(TEST_NC_FILE_2):
                os.remove(TEST_NC_FILE_2)


# noinspection PyAbstractClass
class MyDatasetIO(DatasetIO):

    def __init__(self):
        super().__init__("test")

    @property
    def description(self) -> str:
        return "test"

    @property
    def ext(self) -> str:
        return "test"

    @property
    def modes(self) -> Set[str]:
        return set()

    def fitness(self, input_path: str, path_type: str = None) -> float:
        return 0.0


class DatasetIOTest(unittest.TestCase):

    def test_read_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            ds_io.read('test.nc')

    def test_write_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.write(None, 'test.nc')

    def test_append_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.append(None, 'test.nc')

    def test_insert_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.insert(None, 0, 'test.nc')

    def test_replace_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.replace(None, 0, 'test.nc')

    def test_update_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.update(None, 'test.nc')


class MemDatasetIOTest(unittest.TestCase):

    def test_props(self):
        ds_io = MemDatasetIO()
        self.assertEqual('mem', ds_io.name)
        self.assertEqual('mem', ds_io.ext)
        self.assertEqual('In-memory dataset I/O', ds_io.description)
        self.assertEqual({'r', 'w', 'a'}, ds_io.modes)

    def test_fitness(self):
        ds_io = MemDatasetIO()

        self.assertEqual(0.75, ds_io.fitness("test.mem", path_type=None))
        self.assertEqual(0.75, ds_io.fitness("test.mem", path_type="file"))
        self.assertEqual(0.75, ds_io.fitness("test.mem", path_type="dir"))
        self.assertEqual(0.75, ds_io.fitness("http://dsio/test.mem", path_type="url"))

        self.assertEqual(0.0, ds_io.fitness("test.png", path_type=None))
        self.assertEqual(0.0, ds_io.fitness("test.png", path_type="file"))
        self.assertEqual(0.0, ds_io.fitness("test.png", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("http://dsio/test.png", path_type="url"))

        ds_io.write(xr.Dataset(), "bibo.odod")
        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type=None))
        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type="file"))
        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type="dir"))
        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type="url"))

    def test_read(self):
        ds_io = MemDatasetIO()
        with self.assertRaises(FileNotFoundError):
            ds_io.read('test.nc')
        ds1 = xr.Dataset()
        ds_io._datasets['test.nc'] = ds1
        ds2 = ds_io.read('test.nc')
        self.assertIs(ds2, ds1)

    def test_write(self):
        ds_io = MemDatasetIO()
        ds1 = xr.Dataset()
        ds_io.write(ds1, 'test.nc')
        ds2 = ds_io._datasets['test.nc']
        self.assertIs(ds2, ds1)

    def test_append(self):
        ds_io = MemDatasetIO()
        ds1 = new_test_dataset('2017-02-01', 180, temperature=1.2, precipitation=2.1)
        ds2 = new_test_dataset('2017-02-02', 180, temperature=2.3, precipitation=3.2)
        ds3 = new_test_dataset('2017-02-03', 180, temperature=3.4, precipitation=4.3)
        ds_io.append(ds1, 'test.nc')
        ds_io.append(ds2, 'test.nc')
        ds_io.append(ds3, 'test.nc')
        ds4 = ds_io._datasets.get('test.nc')
        self.assertIsNotNone(ds4)
        self.assertIn('time', ds4)
        self.assertIn('temperature', ds4)
        self.assertEqual(('time', 'lat', 'lon'), ds4.temperature.dims)
        self.assertEqual((3, 180, 360), ds4.temperature.shape)
        expected_time = xr.DataArray(pd.to_datetime(['2017-02-01', '2017-02-02', '2017-02-03']))
        np.testing.assert_equal(expected_time.values, ds4.time.values)


class Netcdf4DatasetIOTest(unittest.TestCase):

    def test_props(self):
        ds_io = Netcdf4DatasetIO()
        self.assertEqual('netcdf4', ds_io.name)
        self.assertEqual('nc', ds_io.ext)
        self.assertEqual('NetCDF-4 file format', ds_io.description)
        self.assertEqual({'a', 'r', 'w'}, ds_io.modes)

    def test_fitness(self):
        ds_io = Netcdf4DatasetIO()

        self.assertEqual(0.875, ds_io.fitness("test.nc", path_type=None))
        self.assertEqual(0.875, ds_io.fitness("test.hdf", path_type=None))
        self.assertEqual(0.875, ds_io.fitness("test.h5", path_type=None))

        self.assertEqual(1.0, ds_io.fitness("test.nc", path_type="file"))
        self.assertEqual(1.0, ds_io.fitness("test.hdf", path_type="file"))
        self.assertEqual(1.0, ds_io.fitness("test.h5", path_type="file"))

        self.assertEqual(0.0, ds_io.fitness("test.nc", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("test.hdf", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("test.h5", path_type="dir"))

        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.nc", path_type="url"))
        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.hdf", path_type="url"))
        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.h5", path_type="url"))

        self.assertEqual(0.125, ds_io.fitness("test.tif", path_type=None))
        self.assertEqual(0.25, ds_io.fitness("test.tif", path_type="file"))
        self.assertEqual(0.0, ds_io.fitness("test.tif", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.tif", path_type="url"))

    def test_read(self):
        ds_io = Netcdf4DatasetIO()
        with self.assertRaises(FileNotFoundError):
            ds_io.read('test.nc')


class ZarrDatasetIOTest(unittest.TestCase):

    def test_props(self):
        ds_io = ZarrDatasetIO()
        self.assertEqual('zarr', ds_io.name)
        self.assertEqual('zarr', ds_io.ext)
        self.assertEqual('Zarr file format (http://zarr.readthedocs.io)', ds_io.description)
        self.assertEqual({'a', 'r', 'w'}, ds_io.modes)

    def test_fitness(self):
        ds_io = ZarrDatasetIO()

        self.assertEqual(0.875, ds_io.fitness("test.zarr", path_type=None))
        self.assertEqual(0.0, ds_io.fitness("test.zarr", path_type="file"))
        self.assertEqual(1.0, ds_io.fitness("test.zarr", path_type="dir"))
        self.assertEqual(0.875, ds_io.fitness("http://dsio/test.zarr", path_type="url"))

        self.assertEqual(0.875, ds_io.fitness("test.zarr.zip", path_type=None))
        self.assertEqual(1.0, ds_io.fitness("test.zarr.zip", path_type="file"))
        self.assertEqual(0.0, ds_io.fitness("test.zarr.zip", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("http://dsio/test.zarr.zip", path_type="url"))

        self.assertEqual(0.0, ds_io.fitness("test.png", path_type=None))
        self.assertEqual(0.0, ds_io.fitness("test.png", path_type="file"))
        self.assertEqual(0.25, ds_io.fitness("test.png", path_type="dir"))
        self.assertEqual(0.125, ds_io.fitness("http://dsio/test.png", path_type="url"))

    def test_read(self):
        ds_io = ZarrDatasetIO()
        with self.assertRaises(ValueError):
            ds_io.read('test.zarr')


class ZarrDatasetS3IOTest(S3Test):

    def test_write_to_and_read_from_s3(self):
        s3_conn = boto3.client('s3', endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        s3_conn.create_bucket(Bucket='upload_bucket', ACL='public-read')

        s3_kwargs = dict(key='test_fake_id', secret='test_fake_secret')
        s3_client_kwargs = {'endpoint_url': MOTO_SERVER_ENDPOINT_URL}

        ds1 = new_cube(width=36, height=18, variables=dict(chl=0.5, tsm=0.2))

        write_cube(ds1,
                   'upload_bucket/cube-1-250-250.zarr',
                   format_name='zarr',
                   s3_kwargs=s3_kwargs,
                   s3_client_kwargs=s3_client_kwargs)

        ds2 = open_cube('upload_bucket/cube-1-250-250.zarr',
                        format_name='zarr',
                        s3_kwargs=s3_kwargs,
                        s3_client_kwargs=s3_client_kwargs)

        self.assertEqual(set(ds1.coords), set(ds2.coords))
        self.assertEqual(set(ds1.data_vars), set(ds2.data_vars))


class CsvDatasetIOTest(unittest.TestCase):

    def test_props(self):
        ds_io = CsvDatasetIO()
        self.assertEqual('csv', ds_io.name)
        self.assertEqual('csv', ds_io.ext)
        self.assertEqual('CSV file format', ds_io.description)
        self.assertEqual({'r', 'w'}, ds_io.modes)

    def test_fitness(self):
        ds_io = CsvDatasetIO()
        self.assertEqual(0.875, ds_io.fitness("test.csv", path_type=None))
        self.assertEqual(1.0, ds_io.fitness("test.csv", path_type="file"))
        self.assertEqual(0.0, ds_io.fitness("test.csv", path_type="dir"))
        self.assertEqual(0.875, ds_io.fitness("http://dsio/test.csv", path_type="url"))

    def test_read(self):
        ds_io = CsvDatasetIO()
        with self.assertRaises(FileNotFoundError):
            ds_io.read('test.csv')


class FindDatasetIOTest(unittest.TestCase):

    def test_find_by_name(self):
        ds_io = find_dataset_io('netcdf4')
        self.assertIsInstance(ds_io, Netcdf4DatasetIO)

        ds_io = find_dataset_io('zarr', modes=['a'])
        self.assertIsInstance(ds_io, ZarrDatasetIO)
        ds_io = find_dataset_io('zarr', modes=['w'])
        self.assertIsInstance(ds_io, ZarrDatasetIO)
        ds_io = find_dataset_io('zarr', modes=['r'])
        self.assertIsInstance(ds_io, ZarrDatasetIO)

        ds_io = find_dataset_io('mem')
        self.assertIsInstance(ds_io, MemDatasetIO)

        ds_io = find_dataset_io('bibo', default=MemDatasetIO())
        self.assertIsInstance(ds_io, MemDatasetIO)

    def test_find_by_ext(self):
        ds_io = find_dataset_io('nc')
        self.assertIsInstance(ds_io, Netcdf4DatasetIO)


class QueryDatasetIOsTest(unittest.TestCase):
    def test_query_dataset_io(self):
        ds_ios = query_dataset_io()
        self.assertEqual(4, len(ds_ios))

    def test_query_dataset_io_with_fn(self):
        ds_ios = query_dataset_io(lambda ds_io: ds_io.name == 'mem')
        self.assertEqual(1, len(ds_ios))
        self.assertIsInstance(ds_ios[0], MemDatasetIO)


class ContextManagerTest(unittest.TestCase):
    def test_it(self):
        class A:
            def __init__(self):
                self.closed = False

            def __enter__(self):
                return self

            def __exit__(self, exception_type, exception_value, traceback):
                self.close()

            def close(self):
                self.closed = True

        def open_a():
            return A()

        with open_a() as a:
            pass

        self.assertEqual(True, a.closed)

        a = open_a()
        self.assertIsInstance(a, A)
        self.assertEqual(False, a.closed)


class GetPathOrObsStoreTest(S3Test):
    def test_path_or_store_read_from_bucket(self):
        s3_store, consolidated = get_path_or_s3_store(
            f'{MOTO_SERVER_ENDPOINT_URL}/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', mode='r')
        self.assertIsInstance(s3_store, fsspec.mapping.FSMap)
        self.assertEqual(False, consolidated)

    def test_path_or_store_write_to_bucket(self):
        s3_conn = boto3.client('s3', endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        s3_conn.create_bucket(Bucket='xcube-examples', ACL='public-read')

        s3_store, consolidated = get_path_or_s3_store(f'{MOTO_SERVER_ENDPOINT_URL}/xcube-examples/fake_cube.zarr',
                                                      s3_client_kwargs={'aws_access_key_id': 'some_fake_id',
                                                                        'aws_secret_access_key': 'some_fake_key'},
                                                      mode='w')
        self.assertIsInstance(s3_store, fsspec.mapping.FSMap)
        self.assertEqual(False, consolidated)

    def test_path_or_store_read_from_local(self):
        path, consolidated = get_path_or_s3_store('../examples/serve/demo/cube-1-250-250.zarr', mode='r')
        self.assertEqual('../examples/serve/demo/cube-1-250-250.zarr', path)
        self.assertEqual(False, consolidated)


class ParseObsUrlAndKwargsTest(unittest.TestCase):
    def test_http(self):
        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs(
            'https://s3.eu-central-1.amazonaws.com/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        self.assertEqual({'key': None, 'secret': None}, s3_kwargs)
        self.assertEqual({'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'}, s3_client_kwargs)

    def test_https_credentials(self):
        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs(
            'https://s3.eu-central-1.amazonaws.com/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', s3_client_kwargs={
                'provider_access_key_id': 'bibo',
                'provider_secret_access_key': '8625345',
            })
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        self.assertEqual({'key': 'bibo', 'secret': '8625345'}, s3_kwargs)
        self.assertEqual({'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'}, s3_client_kwargs)

        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs(
            'https://s3.eu-central-1.amazonaws.com/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', s3_client_kwargs={
                'aws_access_key_id': 'bibo',
                'aws_secret_access_key': '8625345',
            })
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        self.assertEqual({'key': 'bibo', 'secret': '8625345'}, s3_kwargs)
        self.assertEqual({'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'}, s3_client_kwargs)

    def test_s3_schema(self):
        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs('s3://xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                                                                    s3_kwargs={'anon': True},
                                                                    s3_client_kwargs={})
        self.assertEqual('s3://xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        self.assertEqual({'anon': True, 'key': None, 'secret': None}, s3_kwargs)
        self.assertEqual({}, s3_client_kwargs)

    def test_endpoint_url(self):
        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs(
            'eurodatacube-test/demo/xcube-gen-489bb051.zarr',
            s3_kwargs={'anon': True},
            s3_client_kwargs={
                'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'
            }
        )
        self.assertEqual('eurodatacube-test/demo/xcube-gen-489bb051.zarr', root)
        self.assertEqual({'anon': True, 'key': None, 'secret': None}, s3_kwargs)
        self.assertEqual({'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'}, s3_client_kwargs)


class SplitBucketUrlTest(unittest.TestCase):
    def test_http(self):
        endpoint_url, root = split_s3_url(
            'https://s3.eu-central-1.amazonaws.com/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual('https://s3.eu-central-1.amazonaws.com', endpoint_url)
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        endpoint_url, root = split_s3_url('https://xcube-serve:9098/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual('https://xcube-serve:9098', endpoint_url)
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)

    def test_s3(self):
        endpoint_url, root = split_s3_url('s3://xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual(None, endpoint_url)
        self.assertEqual('s3://xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)

    def test_local(self):
        endpoint_url, root = split_s3_url('/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual(None, endpoint_url)
        self.assertEqual('/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)


# test/core/test_dump.py

import unittest

from test.sampledata import new_test_dataset
from xcube.core.dump import dump_dataset


class DumpDatasetTest(unittest.TestCase):
    def test_dump_dataset(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05"],
                                   precipitation=0.4, temperature=275.2)
        for var in dataset.variables.values():
            var.encoding.update({"_FillValue": 999.0})

        print(dataset.dims)

        text = dump_dataset(dataset)
        self.assertIn("<xarray.Dataset>", text)
        self.assertIn("Dimensions:        (time: 5, lat: 180, lon: 360)\n", text)
        self.assertIn("Coordinates:\n", text)
        self.assertIn("  * lon            (lon) float64 ", text)
        self.assertIn("Data variables:\n", text)
        self.assertIn("    precipitation  (time, lat, lon) float64 ", text)
        self.assertNotIn("Encoding for coordinate variable 'lat':\n", text)
        self.assertNotIn("Encoding for data variable 'temperature':\n", text)
        self.assertNotIn("    _FillValue:  999.0\n", text)

        text = dump_dataset(dataset, show_var_encoding=True)
        self.assertIn("<xarray.Dataset>", text)
        self.assertIn("Dimensions:        (time: 5, lat: 180, lon: 360)\n", text)
        self.assertIn("Coordinates:\n", text)
        self.assertIn("  * lon            (lon) float64 ", text)
        self.assertIn("Data variables:\n", text)
        self.assertIn("    precipitation  (time, lat, lon) float64 ", text)
        self.assertIn("Encoding for coordinate variable 'lat':\n", text)
        self.assertIn("Encoding for data variable 'temperature':\n", text)
        self.assertIn("    _FillValue:  999.0\n", text)

        text = dump_dataset(dataset, ["precipitation"])
        self.assertIn("<xarray.DataArray 'precipitation' (time: 5, lat: 180, lon: 360)>\n", text)
        self.assertNotIn("Encoding:\n", text)
        self.assertNotIn("    _FillValue:  999.0", text)

        text = dump_dataset(dataset, ["precipitation"], show_var_encoding=True)
        self.assertIn("<xarray.DataArray 'precipitation' (time: 5, lat: 180, lon: 360)>\n", text)
        self.assertIn("Encoding:\n", text)
        self.assertIn("    _FillValue:  999.0", text)


# test/core/test_edit.py

import unittest

import yaml
import zarr

from test.sampledata import create_highroc_dataset
from xcube.core.new import new_cube
from xcube.core.chunk import chunk_dataset
from xcube.constants import FORMAT_NAME_ZARR
from xcube.core.dsio import rimraf
from xcube.core.edit import edit_metadata
from xcube.core.optimize import optimize_dataset

TEST_CUBE = create_highroc_dataset()

TEST_CUBE_COORDS = chunk_dataset(new_cube(time_periods=3, variables=dict(A=0.5, B=-1.5)),
                                 chunk_sizes=dict(time=1, lat=180, lon=360), format_name=FORMAT_NAME_ZARR)

TEST_CUBE_ZARR = 'test.zarr'

TEST_CUBE_ZARR_COORDS = 'test_coords.zarr'

TEST_CUBE_ZARR_OPTIMIZED = 'test-optimized.zarr'

TEST_CUBE_ZARR_EDIT = 'test_edited_meta.zarr'

TEST_CUBE_ZARR_OPTIMIZED_EDIT = 'test_optimized_edited_meta.zarr'

TEST_NEW_META = {'output_metadata': {'creator_name': 'Brockmann Consult GmbH with love',
                                     'creator_url': 'www.some_very_nice_url.com',
                                     'geospatial_lat_max': 'something around the north pole.'},
                 'conc_chl': {'units': 'happiness'},
                 'some_crazy_var': {'units': 'happiness'}}

TEST_NEW_META_YML = 'test_new_meta.yml'


class EditVariablePropsTest(unittest.TestCase):

    def setUp(self):
        rimraf(TEST_CUBE_ZARR)
        rimraf(TEST_CUBE_ZARR_COORDS)
        rimraf(TEST_NEW_META_YML)
        rimraf(TEST_CUBE_ZARR_EDIT)
        rimraf(TEST_CUBE_ZARR_OPTIMIZED)
        rimraf(TEST_CUBE_ZARR_OPTIMIZED_EDIT)
        TEST_CUBE.to_zarr(TEST_CUBE_ZARR)
        TEST_CUBE_COORDS.to_zarr(TEST_CUBE_ZARR_COORDS)
        with open(TEST_NEW_META_YML, 'w') as outfile:
            yaml.dump(TEST_NEW_META, outfile, default_flow_style=False)

    def tearDown(self):
        rimraf(TEST_CUBE_ZARR)
        rimraf(TEST_CUBE_ZARR_COORDS)
        rimraf(TEST_NEW_META_YML)
        rimraf(TEST_CUBE_ZARR_EDIT)
        rimraf(TEST_CUBE_ZARR_OPTIMIZED)
        rimraf(TEST_CUBE_ZARR_OPTIMIZED_EDIT)

    def test_edit_metadata(self):
        edit_metadata(TEST_CUBE_ZARR, metadata_path=TEST_NEW_META_YML, in_place=False,
                      output_path=TEST_CUBE_ZARR_EDIT, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertNotIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_name', ds2.attrs.keys())

    def test_update_coords_metadata(self):
        edit_metadata(TEST_CUBE_ZARR, metadata_path=TEST_NEW_META_YML, update_coords=True, in_place=False,
                      output_path=TEST_CUBE_ZARR_EDIT, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertIn('geospatial_lon_units', ds2.attrs.keys())
        self.assertEqual('degrees_east', ds2.attrs.__getitem__('geospatial_lon_units'))
        self.assertNotIn('geospatial_lat_max', ds1.attrs.keys())
        self.assertNotIn('geospatial_lat_max', ds2.attrs.keys())

    def test_update_coords_metadata_only(self):
        ds1 = zarr.open(TEST_CUBE_ZARR_COORDS)
        delete_list = ['geospatial_lat_max', 'geospatial_lat_min', 'geospatial_lat_units', 'geospatial_lon_max',
                       'geospatial_lon_min', 'geospatial_lon_units', 'time_coverage_end', 'time_coverage_start']
        for attr in ds1.attrs.keys():
            if attr in delete_list:
                ds1.attrs.__delitem__(attr)
        edit_metadata(TEST_CUBE_ZARR_COORDS, update_coords=True, in_place=False,
                      output_path=TEST_CUBE_ZARR_EDIT, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR_COORDS)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        for attr in delete_list:
            self.assertNotIn(attr, ds1.attrs.keys())
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertIn('geospatial_lat_max', ds2.attrs.keys())
        self.assertIn('geospatial_lat_min', ds2.attrs.keys())
        self.assertIn('geospatial_lat_units', ds2.attrs.keys())
        self.assertIn('geospatial_lon_max', ds2.attrs.keys())
        self.assertEqual(180.0, ds2.attrs.__getitem__('geospatial_lon_max'))
        self.assertEqual(-180.0, ds2.attrs.__getitem__('geospatial_lon_min'))
        self.assertEqual('2010-01-04T00:00:00', ds2.attrs.__getitem__('time_coverage_end'))
        self.assertEqual('2010-01-01T00:00:00', ds2.attrs.__getitem__('time_coverage_start'))
        self.assertEqual('degrees_east', ds2.attrs.__getitem__('geospatial_lon_units'))

    def test_edit_metadata_in_place(self):
        edit_metadata(TEST_CUBE_ZARR, metadata_path=TEST_NEW_META_YML, in_place=True, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR)
        self.assertEqual(36, ds1.__len__())
        self.assertEqual('14-APR-2017 10:27:50.183264', ds1.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds1['conc_chl'].attrs.__getitem__('units'))
        self.assertIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_url', ds1.attrs.keys())
        self.assertEqual('Brockmann Consult GmbH with love', ds1.attrs.__getitem__('creator_name'))
        self.assertEqual('www.some_very_nice_url.com', ds1.attrs.__getitem__('creator_url'))

    def test_edit_zmetadata(self):
        optimize_dataset(TEST_CUBE_ZARR, unchunk_coords=True, output_path=TEST_CUBE_ZARR_OPTIMIZED)
        edit_metadata(TEST_CUBE_ZARR_OPTIMIZED, metadata_path=TEST_NEW_META_YML, in_place=False,
                      output_path=TEST_CUBE_ZARR_OPTIMIZED_EDIT, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.convenience.open_consolidated(TEST_CUBE_ZARR_OPTIMIZED_EDIT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertNotIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_name', ds2.attrs.keys())

    def test_failures(self):
        with self.assertRaises(RuntimeError) as cm:
            edit_metadata('pippo', in_place=True, exception_type=RuntimeError, metadata_path=TEST_NEW_META_YML)
        self.assertEqual('Input path must point to ZARR dataset directory.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            edit_metadata(TEST_CUBE_ZARR, exception_type=RuntimeError, metadata_path=TEST_NEW_META_YML)
        self.assertEqual('Output path must be given.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            edit_metadata(TEST_CUBE_ZARR, output_path=TEST_CUBE_ZARR, exception_type=RuntimeError,
                          metadata_path=TEST_NEW_META_YML)
        self.assertEqual('Output path already exists.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            edit_metadata(TEST_CUBE_ZARR, output_path='./' + TEST_CUBE_ZARR, exception_type=RuntimeError,
                          metadata_path=TEST_NEW_META_YML)
        self.assertEqual('Output path already exists.', f'{cm.exception}')


# test/core/test_evaluate.py

import unittest

import dask.array as da
import numpy as np
import xarray as xr
from xcube.core.evaluate import evaluate_dataset

nan = float('nan')


class EvaluateDatasetTest(unittest.TestCase):
    @classmethod
    def get_test_dataset(cls):
        return xr.Dataset(dict(a=(('y', 'x'), [[0.1, 0.2, 0.4, 0.1], [0.5, 0.1, 0.2, 0.3]]),
                               b=(('y', 'x'), [[0.4, 0.3, 0.2, 0.4], [0.1, 0.2, 0.5, 0.1]],
                                  dict(valid_pixel_expression='a >= 0.2')),
                               c=((), 1.0),
                               d=((), 1.0, dict(expression='a * b'))),
                          coords=dict(x=(('x',), [1, 2, 3, 4]), y=(('y',), [1, 2])),
                          attrs=dict(title='test_compute_dataset')).chunk(dict(x=2, y=2))

    def test_compute_dataset_without_processed_variables(self):
        dataset = self.get_test_dataset()
        computed_dataset = evaluate_dataset(dataset)
        self.assertIsNot(computed_dataset, dataset)
        self.assertIn('x', computed_dataset)
        self.assertIn('y', computed_dataset)
        self.assertIn('a', computed_dataset)
        self.assertIn('b', computed_dataset)
        self.assertIn('c', computed_dataset)
        self.assertIn('d', computed_dataset)
        self.assertIn('x', computed_dataset.coords)
        self.assertIn('y', computed_dataset.coords)
        self.assertIn('title', computed_dataset.attrs)
        self.assertEqual((2, 4), computed_dataset.a.shape)
        self.assertEqual((2, 4), computed_dataset.b.shape)
        self.assertEqual((), computed_dataset.c.shape)
        self.assertNotIn('expression', computed_dataset.c.attrs)
        self.assertEqual((2, 4), computed_dataset.d.shape)
        self.assertIn('expression', computed_dataset.d.attrs)
        np.testing.assert_array_almost_equal(computed_dataset.a.values,
                                             np.array([[0.1, 0.2, 0.4, 0.1], [0.5, 0.1, 0.2, 0.3]]))
        np.testing.assert_array_almost_equal(computed_dataset.b.values,
                                             np.array([[nan, 0.3, 0.2, nan], [0.1, nan, 0.5, 0.1]]))
        np.testing.assert_array_almost_equal(computed_dataset.c.values,
                                             np.array([1.]))
        np.testing.assert_array_almost_equal(computed_dataset.d.values,
                                             np.array([[0.04, 0.06, 0.08, 0.04],
                                                       [0.05, 0.02, 0.1, 0.03]]))

    def test_compute_dataset_with_processed_variables(self):
        dataset = self.get_test_dataset()
        computed_dataset = evaluate_dataset(dataset,
                                            processed_variables=[('a', None),
                                                                 ('b', dict(valid_pixel_expression=None)),
                                                                 ('c', dict(expression='a + b',
                                                                            load=True)),
                                                                 ('d', dict(valid_pixel_expression='c > 0.4',
                                                                            load=True))])
        self.assertIsNot(computed_dataset, dataset)
        self.assertIn('x', computed_dataset)
        self.assertIn('y', computed_dataset)
        self.assertIn('a', computed_dataset)
        self.assertIn('b', computed_dataset)
        self.assertIn('c', computed_dataset)
        self.assertIn('d', computed_dataset)
        self.assertIn('x', computed_dataset.coords)
        self.assertIn('y', computed_dataset.coords)
        self.assertIn('title', computed_dataset.attrs)
        self.assertEqual((2, 4), computed_dataset.a.shape)
        self.assertEqual((2, 4), computed_dataset.b.shape)
        self.assertEqual((2, 4), computed_dataset.c.shape)
        self.assertIsInstance(computed_dataset.a.data, da.Array)
        self.assertIsInstance(computed_dataset.b.data, da.Array)
        self.assertIsInstance(computed_dataset.c.data, np.ndarray)  # load=True --> load c as numpy array
        self.assertIsInstance(computed_dataset.d.data, np.ndarray)  # load=True --> load d as numpy array
        self.assertIn('expression', computed_dataset.c.attrs)
        self.assertEqual((2, 4), computed_dataset.d.shape)
        self.assertIn('expression', computed_dataset.d.attrs)
        np.testing.assert_array_almost_equal(computed_dataset.a.values,
                                             np.array([[0.1, 0.2, 0.4, 0.1], [0.5, 0.1, 0.2, 0.3]]))
        np.testing.assert_array_almost_equal(computed_dataset.b.values,
                                             np.array([[0.4, 0.3, 0.2, 0.4], [0.1, 0.2, 0.5, 0.1]]))
        np.testing.assert_array_almost_equal(computed_dataset.c.values,
                                             np.array([[0.5, 0.5, 0.6, 0.5], [0.6, 0.3, 0.7, 0.4]]))
        np.testing.assert_array_almost_equal(computed_dataset.d.values,
                                             np.array([[0.04, 0.06, 0.08, 0.04], [0.05, nan, 0.1, nan]]))


# test/core/test_extract.py

import unittest

import numpy as np
import pandas as pd
import xarray as xr

from xcube.core.extract import get_cube_point_indexes
from xcube.core.extract import get_cube_values_for_points
from xcube.core.extract import get_dataset_indexes
from xcube.core.new import new_cube


# noinspection PyMethodMayBeStatic
class GetCubeValuesForPointsTest(unittest.TestCase):
    def _new_test_cube(self):
        return new_cube(width=2000,
                        height=1000,
                        x_start=0.0,
                        y_start=50.0,
                        x_res=4.0 / 2000,
                        time_start="2010-01-01",
                        time_periods=20,
                        variables=dict(precipitation=0.6, temperature=276.2))

    def _new_test_points(self):
        return dict(time=np.array(["2010-01-04", "2009-04-09",
                                   "2010-01-08", "2010-01-06",
                                   "2010-01-20", "2010-01-01",
                                   "2010-01-05", "2010-01-03",
                                   ], dtype="datetime64[ns]"),
                    # Note "lat" is a list (see issue #434)
                    lat=[50.0, 51.3, 49.7, 50.1, 51.9, 50.8, 50.2, 52.0],
                    lon=np.array([0.0, 0.1, 0.4, 2.9, 1.6, 0.7, -0.5, 4.0]))

    def test_get_cube_point_values_from_data_frame(self):
        self._test_get_cube_point_values(pd.DataFrame(self._new_test_points()))

    def test_get_cube_point_values_from_dataset(self):
        self._test_get_cube_point_values(xr.Dataset(self._new_test_points()))

    def test_get_cube_point_values_from_dict(self):
        self._test_get_cube_point_values(self._new_test_points())

    def _test_get_cube_point_values(self, points):
        cube = self._new_test_cube()

        expected_prec_values = [0.6, np.nan, np.nan,
                                0.6, 0.6, 0.6, np.nan, 0.6]
        expected_temp_values = [276.2, np.nan, np.nan,
                                276.2, 276.2, 276.2, np.nan, 276.2]
        expected_time_index = [3, -1, 7, 5, 19, 0, 4, 2]
        expected_lat_index = [0, 650, -1, 50, 950, 400, 100, 999]
        expected_lon_index = [0, 50, 200, 1450, 800, 349, -1, 1999]
        values = get_cube_values_for_points(cube, points, include_indexes=True)
        self.assertIsInstance(values, xr.Dataset)
        self.assertEqual(5, len(values.data_vars))
        self.assertEqual({
            'precipitation',
            'temperature',
            'lat_index',
            'lon_index',
            'time_index'},
            set(values.data_vars))
        np.testing.assert_array_almost_equal(np.array(expected_prec_values,
                                                      dtype=np.float64),
                                             values["precipitation"].values)
        np.testing.assert_array_almost_equal(np.array(expected_temp_values,
                                                      dtype=np.float64),
                                             values["temperature"].values)
        np.testing.assert_array_equal(np.array(expected_time_index,
                                               dtype=np.int64),
                                      values["time_index"].values)
        np.testing.assert_array_equal(np.array(expected_lat_index,
                                               dtype=np.int64),
                                      values["lat_index"].values)
        np.testing.assert_array_equal(np.array(expected_lon_index,
                                               dtype=np.int64),
                                      values["lon_index"].values)

    def test_get_cube_point_indexes(self):
        cube = self._new_test_cube()

        expected_time_index = [3., np.nan, 7.,
                               5., 19., 0., 4., 2.]
        expected_lat_index = [0., 650., np.nan,
                              50., 950, 400., 100., 1000. - 1e-9]
        expected_lon_index = [0., 50., 200.,
                              1450., 800., 350., np.nan, 2000. - 1e-9]

        indexes = get_cube_point_indexes(cube, self._new_test_points())
        self.assertIsInstance(indexes, xr.Dataset)
        self.assertEqual(3, len(indexes.data_vars))
        self.assertEqual({'time_index', 'lat_index', 'lon_index'},
                         set(indexes.data_vars))
        np.testing.assert_array_almost_equal(np.array(expected_time_index,
                                                      dtype=np.float64),
                                             indexes["time_index"].values)
        np.testing.assert_array_almost_equal(np.array(expected_lat_index,
                                                      dtype=np.float64),
                                             indexes["lat_index"].values)
        np.testing.assert_array_almost_equal(np.array(expected_lon_index,
                                                      dtype=np.float64),
                                             indexes["lon_index"].values)


# noinspection PyMethodMayBeStatic
class GetDatasetIndexesTest(unittest.TestCase):

    def test_get_dataset_indexes_for_single_cell(self):
        dataset = new_cube(width=360, height=180, drop_bounds=True)
        cell = dataset.isel(time=2, lat=20, lon=30)
        with self.assertRaises(ValueError) as cm:
            get_dataset_indexes(cell, "lon", np.array([-149.5]))
        self.assertEqual("cannot determine cell boundaries"
                         " for coordinate variable 'lon' of size 1",
                         f"{cm.exception}")

    def test_get_dataset_indexes_with_bounds(self):
        dataset = new_cube(width=360, height=180,
                           drop_bounds=False)
        self._assert_get_dataset_indexes_works(dataset)

    def test_get_dataset_indexes_without_bounds(self):
        dataset = new_cube(width=360, height=180,
                           drop_bounds=True)
        self._assert_get_dataset_indexes_works(dataset)

    def test_get_dataset_indexes_with_bounds_inverse_lat(self):
        dataset = new_cube(width=360, height=180,
                           inverse_y=True, drop_bounds=False)
        self._assert_get_dataset_indexes_works(dataset, inverse_lat=True)

    def test_get_dataset_indexes_without_bounds_inverse_lat(self):
        dataset = new_cube(width=360, height=180,
                           inverse_y=True, drop_bounds=True)
        self._assert_get_dataset_indexes_works(dataset, inverse_lat=True)

    def _assert_get_dataset_indexes_works(self, dataset, inverse_lat=False):
        # - lon -------------------------------------

        lon_coords = np.array([-190, -180., -179,
                               -10.4, 0., 10.4, 179., 180.0, 190])
        expected_lon_int64 = np.array([-1, 0, 1,
                                       169, 180, 190,
                                       359, 359, -1],
                                      dtype=np.int64)
        expected_lon_float64 = np.array([np.nan, 0., 1.,
                                         169.6, 180., 190.4,
                                         359., 360., np.nan],
                                        dtype=np.float64)

        indexes = get_dataset_indexes(dataset, "lon", lon_coords,
                                      index_dtype=np.int32)
        np.testing.assert_array_equal(indexes, expected_lon_int64)
        indexes = get_dataset_indexes(dataset, "lon", lon_coords)
        np.testing.assert_array_almost_equal(indexes, expected_lon_float64)

        # - lat -------------------------------------

        lat_coords = np.array([-100, -90., -89,
                               -10.4, 0., 10.4, 89., 90.0, 100])
        if not inverse_lat:
            expected_lat_int64 = np.array([-1, 0, 1,
                                           79, 90, 100, 179, 179, -1],
                                          dtype=np.int64)
            expected_lat_float64 = np.array([np.nan, 0., 1., 79.6,
                                             90., 100.4, 179.,
                                             180., np.nan],
                                            dtype=np.float64)
        else:
            expected_lat_int64 = np.array([-1, 179, 179,
                                           100, 90, 79, 1, 0, -1],
                                          dtype=np.int64)
            expected_lat_float64 = np.array([np.nan, 180., 179.,
                                             100.4, 90., 79.6, 1., 0., np.nan],
                                            dtype=np.float64)

        indexes = get_dataset_indexes(dataset, "lat", lat_coords,
                                      index_dtype=np.int32)
        np.testing.assert_array_equal(indexes, expected_lat_int64)
        indexes = get_dataset_indexes(dataset, "lat", lat_coords)
        np.testing.assert_array_almost_equal(indexes, expected_lat_float64)

        # - time ------------------------------------

        time_coords = np.array(
            ["2010-01-03", "2010-01-02T23:15", "2010-05-15",
             "2010-01-01T12:00", "2010-01-05", "2009-12-07"],
            dtype="datetime64[ns]")
        expected_time_int64 = np.array([2, 1, -1, 0, 4, -1],
                                       dtype=np.int64)
        expected_time_float64 = np.array([2., 1.96875, np.nan,
                                          0.5, 4., np.nan],
                                         dtype=np.float64)

        indexes = get_dataset_indexes(dataset, "time", time_coords,
                                      index_dtype=np.int32)
        np.testing.assert_array_equal(indexes, expected_time_int64)
        indexes = get_dataset_indexes(dataset, "time", time_coords)
        np.testing.assert_array_almost_equal(indexes, expected_time_float64)


# test/core/test_geom.py

import unittest

import geopandas as gpd
import numpy as np
import shapely.geometry
import shapely.wkt
import xarray as xr

from xcube.core.chunk import chunk_dataset
from xcube.core.geom import clip_dataset_by_geometry
from xcube.core.geom import convert_geometry
from xcube.core.geom import get_dataset_bounds
from xcube.core.geom import get_dataset_geometry
from xcube.core.geom import get_geometry_mask
from xcube.core.geom import mask_dataset_by_geometry
from xcube.core.geom import rasterize_features
from xcube.core.new import new_cube


class RasterizeFeaturesIntoDataset(unittest.TestCase):
    def test_rasterize_geodataframe_features_into_dataset_lonlat(self):
        features = self.get_geodataframe_features()
        self._test_rasterize_features(features, 'lon', 'lat')
        self._test_rasterize_features(features, 'lon', 'lat', with_var=True)

    def test_rasterize_geodataframe_features_into_dataset_xy(self):
        features = self.get_geodataframe_features()
        self._test_rasterize_features(features, 'x', 'y')
        self._test_rasterize_features(features, 'x', 'y', with_var=True)

    def test_rasterize_geojson_features_into_dataset_lonlat(self):
        features = self.get_geojson_features()
        self._test_rasterize_features(features, 'lon', 'lat')
        self._test_rasterize_features(features, 'lon', 'lat', with_var=True)

    def test_rasterize_geojson_features_into_dataset_xy(self):
        features = self.get_geojson_features()
        self._test_rasterize_features(features, 'x', 'y')
        self._test_rasterize_features(features, 'x', 'y', with_var=True)

    def _test_rasterize_features(self,
                                 features,
                                 x_name, y_name,
                                 with_var=False):
        dataset = new_cube(
            width=10, height=10,
            x_name=x_name, y_name=y_name,
            x_res=10,
            x_start=-50, y_start=-50,
            variables=dict(d=12.5) if with_var else None
        )
        if with_var:
            dataset['d'] = dataset['d'].chunk({x_name: 4,
                                               y_name: 3,
                                               'time': 1})

        dataset = rasterize_features(dataset,
                                     features,
                                     ['a', 'b', 'c'],
                                     var_props=dict(
                                         b=dict(name='b', dtype=np.float32,
                                                fill_value=np.nan,
                                                attrs=dict(units='meters')),
                                         c=dict(name='c2', dtype=np.uint8,
                                                fill_value=0, converter=int)
                                     ),
                                     in_place=False)
        self.assertIsNotNone(dataset)
        if with_var:
            self.assertEqual({x_name: (4, 4, 2),
                              y_name: (3, 3, 3, 1)},
                             dataset.chunks)
        else:
            self.assertEqual({x_name: (10,),
                              y_name: (10,)},
                             dataset.chunks)
        self.assertIn(x_name, dataset.coords)
        self.assertIn(y_name, dataset.coords)
        self.assertIn('time', dataset.coords)
        self.assertIn('a', dataset)
        self.assertIn('b', dataset)
        self.assertIn('c2', dataset)
        self.assertEquals((10, 10), dataset.a.shape)
        self.assertEquals((10, 10), dataset.b.shape)
        self.assertEquals((10, 10), dataset.c2.shape)
        self.assertEquals(np.float64, dataset.a.dtype)
        self.assertEquals(np.float32, dataset.b.dtype)
        self.assertEquals(np.uint8, dataset.c2.dtype)
        self.assertEquals({}, dataset.a.attrs)
        self.assertEquals({'units': 'meters'}, dataset.b.attrs)
        self.assertEquals({}, dataset.c2.attrs)
        self.assertEquals((y_name, x_name), dataset.a.dims)
        self.assertEquals((y_name, x_name), dataset.b.dims)
        self.assertEquals((y_name, x_name), dataset.c2.dims)
        self.assertEquals(np.array(0.5, dtype=np.float64), dataset.a.min())
        self.assertEquals(np.array(0.8, dtype=np.float64), dataset.a.max())
        self.assertEquals(np.array(2.1, dtype=np.float32), dataset.b.min())
        self.assertEquals(np.array(2.4, dtype=np.float32), dataset.b.max())
        self.assertEquals(np.array(0, dtype=np.uint8), dataset.c2.min())
        self.assertEquals(np.array(9, dtype=np.uint8), dataset.c2.max())
        nan = np.nan
        np.testing.assert_almost_equal(
            np.array([[0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7],
                      [0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7],
                      [0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7],
                      [0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7],
                      [0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7],
                      [0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8],
                      [0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8],
                      [0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8],
                      [0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8],
                      [0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8]]),
            dataset.a.values)

    def get_geodataframe_features(self):
        features = self.get_geojson_features()
        return gpd.GeoDataFrame.from_features(features)

    @staticmethod
    def get_geojson_features():
        feature1 = dict(
            type='Feature',
            geometry=dict(type='Polygon',
                          coordinates=[
                              [(-180, 0), (-1, 0), (-1, 90), (-180, 90),
                               (-180, 0)]]),
            properties=dict(a=0.5, b=2.1, c=9)
        )
        feature2 = dict(
            type='Feature',
            geometry=dict(type='Polygon',
                          coordinates=[
                              [(-180, -90), (-1, -90), (-1, 0), (-180, 0),
                               (-180, -90)]]),
            properties=dict(a=0.6, b=2.2, c=8)
        )
        feature3 = dict(
            type='Feature',
            geometry=dict(type='Polygon',
                          coordinates=[
                              [(20, 0), (180, 0), (180, 90), (20, 90),
                               (20, 0)]]),
            properties=dict(a=0.7, b=2.3, c=7)
        )
        feature4 = dict(
            type='Feature',
            geometry=dict(type='Polygon',
                          coordinates=[
                              [(20, -90), (180, -90), (180, 0), (20, 0),
                               (20, -90)]]),
            properties=dict(a=0.8, b=2.4, c=6)
        )
        return [feature1, feature2, feature3, feature4]


class DatasetGeometryTest(unittest.TestCase):

    def setUp(self) -> None:
        width = 16
        height = 8
        spatial_res = 360 / width
        lon_min = -2 * spatial_res + 0.5 * spatial_res
        lat_min = -2 * spatial_res + 0.5 * spatial_res
        lon_max = lon_min + 6 * spatial_res
        lat_max = lat_min + 3 * spatial_res

        self.triangle = shapely.geometry.Polygon(
            ((lon_min, lat_min),
             (lon_max, lat_min),
             (0.5 * (lon_max + lon_min), lat_max),
             (lon_min, lat_min))
        )

        self.cube = new_cube(width=width,
                             height=height,
                             x_res=spatial_res,
                             drop_bounds=True,
                             variables=dict(temp=273.9, precip=0.9))

    def test_clip_dataset_by_geometry(self):
        cube = clip_dataset_by_geometry(self.cube, self.triangle)
        self._assert_clipped_dataset_has_basic_props(cube)
        cube = clip_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_wkt=True)
        self._assert_clipped_dataset_has_basic_props(cube)
        self._assert_saved_geometry_wkt_is_fine(cube, 'geometry_wkt')
        cube = clip_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_wkt='intersect_geom')
        self._assert_saved_geometry_wkt_is_fine(cube, 'intersect_geom')

    def test_mask_dataset_by_geometry(self):
        cube = mask_dataset_by_geometry(self.cube, self.triangle)
        self._assert_clipped_dataset_has_basic_props(cube)
        cube = mask_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_wkt=True)
        self._assert_saved_geometry_wkt_is_fine(cube, 'geometry_wkt')
        cube = mask_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_wkt='intersect_geom')
        self._assert_saved_geometry_wkt_is_fine(cube, 'intersect_geom')

    def test_mask_dataset_by_geometry_excluded_vars(self):
        cube = mask_dataset_by_geometry(self.cube, self.triangle,
                                        excluded_vars='precip')
        self._assert_clipped_dataset_has_basic_props(cube)

    def test_mask_dataset_by_geometry_store_mask(self):
        cube = mask_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_mask='geom_mask')
        self._assert_clipped_dataset_has_basic_props(cube)
        self._assert_dataset_mask_is_fine(cube, 'geom_mask')

    def test_clip_dataset_for_chunked_input(self):
        cube = chunk_dataset(self.cube,
                             chunk_sizes=dict(time=1, lat=90, lon=90))
        cube = clip_dataset_by_geometry(cube, self.triangle)
        self._assert_clipped_dataset_has_basic_props(cube)
        self.assertEqual(((1, 1, 1, 1, 1), (4,), (7,)), cube.temp.chunks)
        self.assertEqual(((1, 1, 1, 1, 1), (4,), (7,)), cube.precip.chunks)

    def test_mask_dataset_for_chunked_input(self):
        cube = chunk_dataset(self.cube,
                             chunk_sizes=dict(time=1, lat=90, lon=90))
        cube = mask_dataset_by_geometry(cube, self.triangle)
        self._assert_clipped_dataset_has_basic_props(cube)
        self.assertEqual(((1, 1, 1, 1, 1), (4,), (7,)), cube.temp.chunks)
        self.assertEqual(((1, 1, 1, 1, 1), (4,), (7,)), cube.precip.chunks)

    def _assert_clipped_dataset_has_basic_props(self, dataset):
        self.assertEqual({'time': 5, 'lat': 4, 'lon': 7}, dataset.dims)
        self.assertIn('temp', dataset)
        self.assertIn('precip', dataset)
        temp = dataset['temp']
        precip = dataset['precip']
        self.assertEqual(('time', 'lat', 'lon'), temp.dims)
        self.assertEqual((5, 4, 7), temp.shape)
        self.assertEqual(('time', 'lat', 'lon'), precip.dims)
        self.assertEqual((5, 4, 7), precip.shape)

        self.assertIn('geospatial_lon_min', dataset.attrs)
        self.assertIn('geospatial_lon_max', dataset.attrs)
        self.assertIn('geospatial_lon_units', dataset.attrs)
        self.assertIn('geospatial_lon_resolution', dataset.attrs)
        self.assertIn('geospatial_lat_min', dataset.attrs)
        self.assertIn('geospatial_lat_max', dataset.attrs)
        self.assertIn('geospatial_lat_units', dataset.attrs)
        self.assertIn('geospatial_lat_resolution', dataset.attrs)
        self.assertIn('time_coverage_start', dataset.attrs)
        self.assertIn('time_coverage_end', dataset.attrs)
        self.assertIn('date_modified', dataset.attrs)

    def _assert_dataset_mask_is_fine(self, dataset, mask_var_name):
        self.assertIn(mask_var_name, dataset)
        geom_mask = dataset[mask_var_name]
        self.assertEqual(('lat', 'lon'), geom_mask.dims)
        self.assertEqual((4, 7), geom_mask.shape)
        np.testing.assert_array_almost_equal(
            geom_mask.values.astype(np.byte),
            np.array([[0, 0, 0, 0, 0, 0, 0],
                      [0, 0, 0, 1, 1, 0, 0],
                      [0, 0, 1, 1, 1, 1, 0],
                      [0, 1, 1, 1, 1, 1, 1]], dtype=np.byte)
        )

    def _assert_saved_geometry_wkt_is_fine(self, dataset, geometry_wkt_name):
        self.assertIn(geometry_wkt_name, dataset.attrs)
        actual = shapely.wkt.loads(dataset.attrs[geometry_wkt_name])
        expected = shapely.wkt.loads(
            'POLYGON ((-33.75 -33.75, 33.75 33.75,'
            ' 101.25 -33.75, -33.75 -33.75))'
        )
        self.assertTrue(actual.difference(expected).is_empty)


class GetGeometryMaskTest(unittest.TestCase):
    # noinspection PyMethodMayBeStatic
    def test_get_geometry_mask(self):
        w = 16
        h = 8
        res = 1.0
        lon_min = 0
        lat_min = 0
        lon_max = lon_min + w * res
        lat_max = lat_min + h * res

        triangle = shapely.geometry.Polygon(((lon_min, lat_min),
                                             (lon_max, lat_min),
                                             (lon_max, lat_max),
                                             (lon_min, lat_min)))

        actual_mask = get_geometry_mask(w, h, triangle, lon_min,
                                        lat_min, res, all_touched=True)
        expected_mask = np.array(
            [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype=np.byte
        )
        np.testing.assert_array_almost_equal(expected_mask,
                                             actual_mask.astype('byte'))

        actual_mask = get_geometry_mask(w, h, triangle, lon_min,
                                        lat_min, res, all_touched=False)
        expected_mask = np.array(
            [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype=np.byte
        )
        np.testing.assert_array_almost_equal(expected_mask,
                                             actual_mask.astype('byte'))

        smaller_triangle = triangle.buffer(-1.5 * res)
        actual_mask = get_geometry_mask(w, h, smaller_triangle, lon_min,
                                        lat_min, res, all_touched=True)
        expected_mask = np.array(
            [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.byte
        )
        np.testing.assert_array_almost_equal(expected_mask, actual_mask)

        actual_mask = get_geometry_mask(w, h, smaller_triangle, lon_min,
                                        lat_min, res, all_touched=False)
        expected_mask = np.array(
            [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.byte
        )
        np.testing.assert_array_almost_equal(expected_mask, actual_mask)


class GetDatasetGeometryTest(unittest.TestCase):

    def test_nominal(self):
        ds1, ds2 = _get_nominal_datasets()
        bounds = get_dataset_geometry(ds1)
        self.assertEqual(
            shapely.geometry.box(-25.0, -15.0, 15.0, 15.0), bounds
        )
        bounds = get_dataset_geometry(ds2)
        self.assertEqual(
            shapely.geometry.box(-25.0, -15.0, 15.0, 15.0), bounds
        )

    def test_inv_y(self):
        ds1, ds2 = _get_inv_y_datasets()
        bounds = get_dataset_geometry(ds1)
        self.assertEqual(
            shapely.geometry.box(-25.0, -15.0, 15.0, 15.0), bounds
        )
        bounds = get_dataset_geometry(ds2)
        self.assertEqual(
            shapely.geometry.box(-25.0, -15.0, 15.0, 15.0), bounds
        )

    def test_antimeridian(self):
        ds1, ds2 = _get_antimeridian_datasets()
        bounds = get_dataset_geometry(ds1)
        self.assertEqual(shapely.geometry.MultiPolygon(
            polygons=[
                shapely.geometry.box(165.0, -15.0, 180.0, 15.0),
                shapely.geometry.box(-180.0, -15.0, -155.0, 15.0)
            ]),
            bounds)
        bounds = get_dataset_geometry(ds2)
        self.assertEqual(shapely.geometry.MultiPolygon(
            polygons=[
                shapely.geometry.box(165.0, -15.0, 180.0, 15.0),
                shapely.geometry.box(-180.0, -15.0, -155.0, 15.0)
            ]),
            bounds)


class GetDatasetBoundsTest(unittest.TestCase):
    def test_nominal(self):
        ds1, ds2 = _get_nominal_datasets()
        bounds = get_dataset_bounds(ds1)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
        bounds = get_dataset_bounds(ds2)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)

    def test_inv_y(self):
        ds1, ds2 = _get_inv_y_datasets()
        bounds = get_dataset_bounds(ds1)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
        bounds = get_dataset_bounds(ds2)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)

    def test_inv_y_wrong_order_bounds(self):
        ds1, ds2 = _get_inv_y_wrong_order_bounds_datasets()
        bounds = get_dataset_bounds(ds1)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
        bounds = get_dataset_bounds(ds2)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)

    def test_anti_meridian(self):
        ds1, ds2 = _get_antimeridian_datasets()
        bounds = get_dataset_bounds(ds1)
        self.assertEqual((165.0, -15.0, -155.0, 15.0), bounds)
        bounds = get_dataset_bounds(ds2)
        self.assertEqual((165.0, -15.0, -155.0, 15.0), bounds)


def _get_nominal_datasets():
    data_vars = dict(a=(("time", "lat", "lon"), np.random.rand(5, 3, 4)))

    coords = dict(time=(("time",), np.array(range(0, 5))),
                  lat=(("lat",), np.array([-10, 0., 10])),
                  lon=(("lon",), np.array([-20, -10, 0., 10])))
    ds1 = xr.Dataset(coords=coords, data_vars=data_vars)

    coords.update(dict(lat_bnds=(("lat", "bnds"), np.array([[-15, -5], [-5., 5], [5, 15]])),
                       lon_bnds=(
                           ("lon", "bnds"), np.array([[-25., -15.], [-15., -5.], [-5., 5.], [5., 15.]]))
                       ))
    ds2 = xr.Dataset(coords=coords, data_vars=data_vars)

    return ds1, ds2


def _get_inv_y_datasets():
    ds1, ds2 = _get_nominal_datasets()
    ds1 = ds1.assign_coords(lat=(("lat",), ds1.lat.values[::-1]))
    ds2 = ds2.assign_coords(lat=(("lat",), ds1.lat.values))
    ds2 = ds2.assign_coords(lat_bnds=(("lat", "bnds"),
                                      ds2.lat_bnds.values[::-1, ::-1]))
    return ds1, ds2


def _get_inv_y_wrong_order_bounds_datasets():
    ds1, ds2 = _get_nominal_datasets()
    ds1 = ds1.assign_coords(lat=(("lat",), ds1.lat.values[::-1]))
    ds2 = ds2.assign_coords(lat=(("lat",), ds1.lat.values))
    ds2 = ds2.assign_coords(lat_bnds=(("lat", "bnds"),
                                      ds2.lat_bnds.values[::-1, ::]))
    return ds1, ds2


def _get_antimeridian_datasets():
    ds1, ds2 = _get_nominal_datasets()
    ds1 = ds1.assign_coords(lon=(("lon",),
                                 np.array([170., 180., -170., -160.])))
    ds2 = ds2.assign_coords(lon=(("lon",),
                                 ds1.lon.values))
    ds2 = ds2.assign_coords(
        lon_bnds=(("lon", "bnds"),
                  np.array([[165., 175], [175., -175.],
                            [-175., -165], [-165., -155.]])))
    return ds1, ds2


class ConvertGeometryTest(unittest.TestCase):
    def test_convert_null(self):
        self.assertIs(None, convert_geometry(None))

    def test_convert_to_point(self):
        expected_point = shapely.geometry.Point(12.8, -34.4)
        self.assertIs(expected_point,
                      convert_geometry(expected_point))
        self.assertEqual(expected_point,
                         convert_geometry([12.8, -34.4]))
        self.assertEqual(expected_point,
                         convert_geometry(np.array([12.8, -34.4])))
        self.assertEqual(expected_point,
                         convert_geometry(expected_point.wkt))
        self.assertEqual(expected_point,
                         convert_geometry(expected_point.__geo_interface__))

    def test_convert_box_as_point(self):
        expected_point = shapely.geometry.Point(12.8, -34.4)
        self.assertEqual(expected_point,
                         convert_geometry([12.8, -34.4, 12.8, -34.4]))

    def test_convert_to_box(self):
        expected_box = shapely.geometry.box(12.8, -34.4, 14.2, 20.6)
        self.assertIs(expected_box,
                      convert_geometry(expected_box))
        self.assertEqual(expected_box,
                         convert_geometry([12.8, -34.4, 14.2, 20.6]))
        self.assertEqual(expected_box,
                         convert_geometry(
                             np.array([12.8, -34.4, 14.2, 20.6])
                         ))
        self.assertEqual(expected_box,
                         convert_geometry(expected_box.wkt))
        self.assertEqual(expected_box,
                         convert_geometry(expected_box.__geo_interface__))

    def test_convert_to_split_box(self):
        expected_split_box = shapely.geometry.MultiPolygon(polygons=[
            shapely.geometry.Polygon(
                ((180.0, -34.4), (180.0, 20.6), (172.1, 20.6), (172.1, -34.4),
                 (180.0, -34.4))
            ),
            shapely.geometry.Polygon(
                ((-165.7, -34.4), (-165.7, 20.6), (-180.0, 20.6),
                 (-180.0, -34.4),
                 (-165.7, -34.4)))]
        )
        self.assertEqual(expected_split_box,
                         convert_geometry([172.1, -34.4, -165.7, 20.6]))

    def test_convert_from_geojson_feature_dict(self):
        expected_box1 = shapely.geometry.box(-10, -20, 20, 10)
        expected_box2 = shapely.geometry.box(30, 20, 50, 40)
        feature1 = dict(type='Feature',
                        geometry=expected_box1.__geo_interface__)
        feature2 = dict(type='Feature',
                        geometry=expected_box2.__geo_interface__)
        feature_collection = dict(type='FeatureCollection',
                                  features=(feature1, feature2))

        actual_geom = convert_geometry(feature1)
        self.assertEqual(expected_box1, actual_geom)

        actual_geom = convert_geometry(feature2)
        self.assertEqual(expected_box2, actual_geom)

        expected_geom = shapely.geometry.GeometryCollection(
            geoms=[expected_box1, expected_box2]
        )
        actual_geom = convert_geometry(feature_collection)
        self.assertEqual(expected_geom, actual_geom)

    def test_convert_invalid_box(self):
        from xcube.core.geom import _INVALID_BOX_COORDS_MSG

        with self.assertRaises(ValueError) as cm:
            convert_geometry([12.8, 20.6, 14.2, -34.4])
        self.assertEqual(_INVALID_BOX_COORDS_MSG, f'{cm.exception}')
        with self.assertRaises(ValueError) as cm:
            convert_geometry([12.8, -34.4, 12.8, 20.6])
        self.assertEqual(_INVALID_BOX_COORDS_MSG, f'{cm.exception}')
        with self.assertRaises(ValueError) as cm:
            convert_geometry([12.8, -34.4, 12.8, 20.6])
        self.assertEqual(_INVALID_BOX_COORDS_MSG, f'{cm.exception}')

    def test_invalid(self):
        from xcube.core.geom import _INVALID_GEOMETRY_MSG

        with self.assertRaises(ValueError) as cm:
            convert_geometry(dict(coordinates=[12.8, -34.4]))
        self.assertEqual(_INVALID_GEOMETRY_MSG, f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            convert_geometry([12.8, -34.4, '?'])
        self.assertEqual(_INVALID_GEOMETRY_MSG, f'{cm.exception}')


# test/core/test_level.py

import os
import time
import unittest

import numpy as np
import xarray as xr

from xcube.core.dsio import rimraf
from xcube.core.level import compute_levels
from xcube.core.level import read_levels
from xcube.core.level import write_levels


def get_path(filename):
    return os.path.join(os.path.dirname(__file__), filename)


class PyramidTest(unittest.TestCase):
    @classmethod
    def create_test_dataset(cls, shape, chunks=None):
        size = int(np.prod(shape))
        dims = ["time", "y", "x"]
        a_data = np.linspace(0, 1, size, dtype=np.float64).reshape(shape)
        a = xr.DataArray(a_data, dims=dims)
        b_data = np.linspace(-1, 0, size, dtype=np.float64).reshape(shape)
        b = xr.DataArray(b_data, dims=dims)
        if chunks:
            a.encoding.update(chunks=chunks, chunksizes=chunks)
            b.encoding.update(chunks=chunks, chunksizes=chunks)
            return xr.Dataset(dict(a=a, b=b)) \
                .chunk(chunks={dims[i]: chunks[i]
                               for i in range(len(dims))})
        else:
            return xr.Dataset(dict(a=a, b=b))

    def test_compute_levels(self):
        dataset = self.create_test_dataset(shape=(5, 200, 400),
                                           chunks=(1, 25, 25))
        levels = compute_levels(dataset)
        self._assert_levels_ok(levels,
                               expected_num_levels=4,
                               expected_shapes=[
                                   (5, 200, 400),
                                   (5, 100, 200),
                                   (5, 50, 100),
                                   (5, 25, 50),
                               ],
                               expected_chunks=[
                                   ((1,) * 5, (25,) * 8, (25,) * 16),
                                   ((1,) * 5, (25,) * 4, (25,) * 8),
                                   ((1,) * 5, (25,) * 2, (25,) * 4),
                                   ((1,) * 5, (25,) * 1, (25,) * 2),
                               ])

    def test_compute_levels_with_max_levels(self):
        dataset = self.create_test_dataset(shape=(5, 200, 400),
                                           chunks=(1, 25, 25))
        levels = compute_levels(dataset, num_levels_max=3)
        self._assert_levels_ok(levels,
                               expected_num_levels=3,
                               expected_shapes=[
                                   (5, 200, 400),
                                   (5, 100, 200),
                                   (5, 50, 100),
                               ],
                               expected_chunks=[
                                   ((1,) * 5, (25,) * 8, (25,) * 16),
                                   ((1,) * 5, (25,) * 4, (25,) * 8),
                                   ((1,) * 5, (25,) * 2, (25,) * 4),
                               ])

    def test_write_read_levels_with_even_sizes(self):
        shape = (5, 200, 400)
        tile_shape = (25, 25)
        expected_num_levels = 4
        expected_shapes = [
            (5, 200, 400),
            (5, 100, 200),
            (5, 50, 100),
            (5, 25, 50),
        ]
        expected_chunks = [
            ((1,) * 5, (25,) * 8, (25,) * 16),
            ((1,) * 5, (25,) * 4, (25,) * 8),
            ((1,) * 5, (25,) * 2, (25,) * 4),
            ((1,) * 5, (25,) * 1, (25,) * 2),
        ]
        self._assert_io_ok(shape,
                           tile_shape,
                           False,
                           expected_num_levels,
                           expected_shapes,
                           expected_chunks)
        self._assert_io_ok(shape,
                           tile_shape,
                           True,
                           expected_num_levels,
                           expected_shapes,
                           expected_chunks)

    def test_write_read_levels_with_odd_sizes(self):
        shape = (5, 203, 405)
        tile_shape = (37, 38)
        expected_num_levels = 3
        expected_shapes = [
            (5, 203, 405),
            (5, 102, 203),
            (5, 51, 102),
        ]
        expected_chunks = [
            ((1,) * 5,
             (37, 37, 37, 37, 37, 18),
             (38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 25)),
            ((1,) * 5,
             (37, 37, 28),
             (38, 38, 38, 38, 38, 13)),
            ((1,) * 5,
             (37, 14),
             (38, 38, 26)),
        ]
        self._assert_io_ok(shape,
                           tile_shape,
                           False,
                           expected_num_levels,
                           expected_shapes,
                           expected_chunks)
        self._assert_io_ok(shape,
                           tile_shape,
                           True,
                           expected_num_levels,
                           expected_shapes,
                           expected_chunks)

    def _assert_io_ok(self,
                      shape,
                      tile_shape,
                      link_input: bool,
                      expected_num_levels,
                      expected_shapes,
                      expected_chunks):

        input_path = get_path("pyramid-input.zarr")
        output_path = get_path("pyramid-output")

        rimraf(input_path)
        rimraf(output_path)

        try:
            dataset = self.create_test_dataset(shape,
                                               chunks=(1, *tile_shape))
            dataset.to_zarr(input_path)

            t0 = time.perf_counter()

            levels = write_levels(output_path,
                                  dataset=dataset,
                                  spatial_tile_shape=tile_shape,
                                  input_path=input_path,
                                  link_input=link_input)

            print(f"write time total: ", time.perf_counter() - t0)

            self._assert_levels_ok(levels,
                                   expected_num_levels,
                                   expected_shapes,
                                   expected_chunks)

            t0 = time.perf_counter()

            levels = read_levels(output_path)

            print(f"read time total: ", time.perf_counter() - t0)

            self._assert_levels_ok(levels,
                                   expected_num_levels,
                                   expected_shapes,
                                   expected_chunks)

        finally:
            rimraf(input_path)
            rimraf(output_path)

    def _assert_levels_ok(self,
                          levels,
                          expected_num_levels,
                          expected_shapes,
                          expected_chunks):
        self.assertIsInstance(levels, list)
        self.assertEqual(expected_num_levels, len(levels))

        for i in range(expected_num_levels):
            msg = f"at index {i}"
            self.assertIsInstance(levels[i], xr.Dataset, msg=msg)
            self.assertEqual(2, len(levels[i].data_vars), msg=msg)
            self.assertIn("a", levels[i], msg=msg)
            self.assertIn("b", levels[i], msg=msg)

        for i in range(expected_num_levels):
            msg = f"at index {i}"
            self.assertEqual(expected_shapes[i], levels[i]["a"].shape,
                             msg=msg)
            self.assertEqual(expected_shapes[i], levels[i]["b"].shape,
                             msg=msg)

        for i in range(expected_num_levels):
            msg = f"at index {i}"
            self.assertEqual(expected_chunks[i], levels[i]["a"].chunks,
                             msg=msg)
            self.assertEqual(expected_chunks[i], levels[i]["b"].chunks,
                             msg=msg)


# test/core/test_maskset.py

import unittest
from typing import Tuple

import numpy as np
import xarray as xr
from numpy.testing import assert_array_almost_equal

from test.sampledata import create_highroc_dataset, create_c2rcc_flag_var, create_cmems_sst_flag_var
from xcube.core.maskset import MaskSet


class MaskSetTest(unittest.TestCase):
    def test_mask_set_with_flag_mask_str(self):
        flag_var = create_cmems_sst_flag_var().chunk(dict(lon=2, lat=2))
        mask_set = MaskSet(flag_var)

        self.assertEqual('mask(sea=(1, None), land=(2, None), lake=(4, None), ice=(8, None))',
                         str(mask_set))

        expected_chunks = ((1,), (2, 1), (2, 2))
        self.assertMaskOk(mask_set, 'sea', expected_chunks)
        self.assertMaskOk(mask_set, 'land', expected_chunks)
        self.assertMaskOk(mask_set, 'lake', expected_chunks)
        self.assertMaskOk(mask_set, 'ice', expected_chunks)

        validation_data = ((0, 'sea', mask_set.sea, np.array([[[1, 0, 0, 0],
                                                               [1, 1, 0, 0],
                                                               [1, 1, 1, 0]]],
                                                             dtype=np.uint8)),
                           (1, 'land', mask_set.land, np.array([[[0, 1, 0, 0],
                                                                 [0, 0, 1, 1],
                                                                 [0, 0, 0, 1]]],
                                                               dtype=np.uint8)),
                           (2, 'lake', mask_set.lake, np.array([[[0, 0, 1, 1],
                                                                 [0, 0, 0, 0],
                                                                 [0, 0, 0, 0]]],
                                                               dtype=np.uint8)),
                           (3, 'ice', mask_set.ice, np.array([[[1, 1, 1, 0],
                                                               [1, 0, 0, 0],
                                                               [0, 0, 0, 0]]],
                                                             dtype=np.uint8)))

        for index, name, mask, data in validation_data:
            self.assertIs(mask, mask_set[index])
            self.assertIs(mask, mask_set[name])
            assert_array_almost_equal(mask.values, data, err_msg=f'{index}, {name}, {mask.name}')

    def test_mask_set_with_flag_mask_int_array(self):
        flag_var = create_c2rcc_flag_var().chunk(dict(x=2, y=2))
        mask_set = MaskSet(flag_var)

        self.assertEqual('c2rcc_flags(F1=(1, None), F2=(2, None), F3=(4, None), F4=(8, None))',
                         str(mask_set))

        expected_chunks = ((2, 1), (2, 2))
        self.assertMaskOk(mask_set, 'F1', expected_chunks)
        self.assertMaskOk(mask_set, 'F2', expected_chunks)
        self.assertMaskOk(mask_set, 'F3', expected_chunks)
        self.assertMaskOk(mask_set, 'F4', expected_chunks)

        validation_data = ((0, 'F1', mask_set.F1, np.array([[1, 1, 1, 1],
                                                            [1, 0, 1, 0],
                                                            [0, 1, 1, 1]],
                                                           dtype=np.uint8)),
                           (1, 'F2', mask_set.F2, np.array([[0, 0, 0, 0],
                                                            [0, 0, 0, 1],
                                                            [0, 0, 0, 0]],
                                                           dtype=np.uint8)),
                           (2, 'F3', mask_set.F3, np.array([[0, 0, 0, 0],
                                                            [0, 1, 0, 0],
                                                            [0, 0, 0, 0]],
                                                           dtype=np.uint8)),
                           (3, 'F4', mask_set.F4, np.array([[0, 0, 0, 0],
                                                            [0, 0, 0, 0],
                                                            [1, 0, 0, 0]],
                                                           dtype=np.uint8)))

        for index, name, mask, data in validation_data:
            self.assertIs(mask, mask_set[index])
            self.assertIs(mask, mask_set[name])
            assert_array_almost_equal(mask.values, data)

    def assertMaskOk(self,
                     mask_set: MaskSet,
                     mask_name: str,
                     expected_chunks: Tuple[Tuple[int, ...], ...] = None):

        mask = getattr(mask_set, mask_name)

        self.assertIsInstance(mask, xr.DataArray)

        if expected_chunks:
            import dask.array as da
            self.assertIsInstance(mask.data, da.Array)
        self.assertEqual(expected_chunks, mask.chunks)

        # assert same instance is returned
        self.assertIs(mask, getattr(mask_set, mask_name))

        self.assertIn(mask_name, mask_set)
        # assert same instance is returned
        self.assertIs(mask, mask_set[mask_name])

    def test_get_mask_sets(self):
        dataset = create_highroc_dataset()
        mask_sets = MaskSet.get_mask_sets(dataset)
        self.assertIsNotNone(mask_sets)
        self.assertEqual(len(mask_sets), 1)
        self.assertIn('c2rcc_flags', mask_sets)
        mask_set = mask_sets['c2rcc_flags']
        self.assertIsInstance(mask_set, MaskSet)

    def test_mask_set_with_flag_values(self):
        s2l2a_slc_meanings = ['no_data',
                              'saturated_or_defective',
                              'dark_area_pixels',
                              'cloud_shadows',
                              'vegetation',
                              'bare_soils',
                              'water',
                              'clouds_low_probability_or_unclassified',
                              'clouds_medium_probability',
                              'clouds_high_probability',
                              'cirrus',
                              'snow_or_ice']

        data = np.array([[1, 2, 8, 3],
                         [7, 6, 0, 4],
                         [9, 5, 11, 10]], dtype=np.uint8)
        flag_var = xr.DataArray(data,
                                dims=('y', 'x'),
                                name='SLC',
                                attrs=dict(
                                    long_name="Scene classification flags",
                                    flag_values=','.join(f'{i}' for i in range(len(s2l2a_slc_meanings))),
                                    flag_meanings=' '.join(s2l2a_slc_meanings),
                                ))

        mask_set = MaskSet(flag_var)

        self.assertEqual('SLC(no_data=(None, 0), saturated_or_defective=(None, 1), dark_area_pixels=(None, 2), '
                         'cloud_shadows=(None, 3), vegetation=(None, 4), bare_soils=(None, 5), water=(None, 6), '
                         'clouds_low_probability_or_unclassified=(None, 7), clouds_medium_probability=(None, 8), '
                         'clouds_high_probability=(None, 9), cirrus=(None, 10), snow_or_ice=(None, 11))',
                         str(mask_set))

        validation_data = ((0, 'no_data', mask_set.no_data, np.array([[0, 0, 0, 0],
                                                                      [0, 0, 1, 0],
                                                                      [0, 0, 0, 0]],
                                                                     dtype=np.uint8)),
                           (4, 'vegetation', mask_set.vegetation, np.array([[0, 0, 0, 0],
                                                                            [0, 0, 0, 1],
                                                                            [0, 0, 0, 0]],
                                                                           dtype=np.uint8)),
                           (10, 'cirrus', mask_set.cirrus, np.array([[0, 0, 0, 0],
                                                                     [0, 0, 0, 0],
                                                                     [0, 0, 0, 1]],
                                                                    dtype=np.uint8)),
                           (6, 'water', mask_set.water, np.array([[0, 0, 0, 0],
                                                                  [0, 1, 0, 0],
                                                                  [0, 0, 0, 0]],
                                                                 dtype=np.uint8)))

        for index, name, mask, data in validation_data:
            msg = f'index={index}, name={name!r}, data={data!r}'
            self.assertIs(mask, mask_set[index], msg=msg)
            self.assertIs(mask, mask_set[name], msg=msg)
            assert_array_almost_equal(mask.values, data, err_msg=msg)

        self.assertEqual(set(s2l2a_slc_meanings), set(dir(mask_set)))

        html = mask_set._repr_html_()
        self.assertTrue(html.startswith('<html>'))
        self.assertTrue(html.endswith('</html>'))

    def test_mask_set_with_missing_values_and_masks_attrs(self):
        flag_var = create_c2rcc_flag_var().chunk(dict(x=2, y=2))
        flag_var.attrs.pop('flag_masks', None)
        flag_var.attrs.pop('flag_values', None)
        with self.assertRaises(ValueError):
            MaskSet(flag_var)

    def test_mask_set_with_missing_meanings_attr(self):
        flag_var = create_c2rcc_flag_var().chunk(dict(x=2, y=2))
        flag_var.attrs.pop('flag_meanings', None)
        with self.assertRaises(ValueError):
            MaskSet(flag_var)


# test/core/test_mldataset.py

import os
import unittest

import numpy as np
import pandas as pd
import s3fs
import xarray as xr

from test.s3test import S3Test, MOTO_SERVER_ENDPOINT_URL
from xcube.core.dsio import write_dataset
from xcube.core.geom import clip_dataset_by_geometry
from xcube.core.mldataset import BaseMultiLevelDataset
from xcube.core.mldataset import CombinedMultiLevelDataset
from xcube.core.mldataset import ComputedMultiLevelDataset
from xcube.core.mldataset import ObjectStorageMultiLevelDataset
from xcube.core.mldataset import open_ml_dataset_from_object_storage
from xcube.core.mldataset import write_levels
from xcube.util.tilegrid import ImageTileGrid


class CombinedMultiLevelDatasetTest(unittest.TestCase):
    def test_it(self):
        ml_ds_1 = BaseMultiLevelDataset(
            _get_test_dataset(('noise_1', 'noise_2'))
        )
        ml_ds_2 = BaseMultiLevelDataset(
            _get_test_dataset(('noise_3', 'noise_4'))
        )
        ml_ds_3 = BaseMultiLevelDataset(
            _get_test_dataset(('noise_5', 'noise_6'))
        )

        ml_ds = CombinedMultiLevelDataset([ml_ds_1, ml_ds_2, ml_ds_3])

        self.assertEqual(3, ml_ds.num_levels)
        self.assertIsInstance(ml_ds.tile_grid, ImageTileGrid)
        self.assertEqual((180, 180), ml_ds.tile_grid.tile_size)
        self.assertEqual(3, ml_ds.tile_grid.num_levels)

        expected_var_names = {'noise_1', 'noise_2',
                              'noise_3', 'noise_4',
                              'noise_5', 'noise_6'}

        ds0 = ml_ds.get_dataset(0)
        self.assertEqual({'time': 14, 'lat': 720, 'lon': 1440, 'bnds': 2},
                         ds0.dims)
        self.assertEqual(expected_var_names, set(map(str, ds0.data_vars)))
        self.assertTrue(all(v.dims == ('time', 'lat', 'lon')
                            for v in ds0.data_vars.values()))

        ds1 = ml_ds.get_dataset(1)
        self.assertEqual({'time': 14, 'lat': 360, 'lon': 720},
                         ds1.dims)
        self.assertEqual(expected_var_names, set(map(str, ds1.data_vars)))
        self.assertTrue(all(v.dims == ('time', 'lat', 'lon')
                            for v in ds1.data_vars.values()))

        ds2 = ml_ds.get_dataset(2)
        self.assertEqual({'time': 14, 'lat': 180, 'lon': 360},
                         ds2.dims)
        self.assertEqual(expected_var_names, set(map(str, ds2.data_vars)))
        self.assertTrue(all(v.dims == ('time', 'lat', 'lon')
                            for v in ds2.data_vars.values()))

        self.assertEqual([ds0, ds1, ds2], ml_ds.datasets)

        ml_ds.close()


class BaseMultiLevelDatasetTest(unittest.TestCase):
    def test_basic_props(self):
        ds = _get_test_dataset()
        ml_ds = BaseMultiLevelDataset(ds)

        self.assertIsInstance(ml_ds.ds_id, str)

        self.assertEqual(3, ml_ds.num_levels)
        self.assertEqual([(0.25, 0.25), (0.5, 0.5), (1.0, 1.0)],
                         ml_ds.resolutions)

        tile_grid = ml_ds.tile_grid
        self.assertIsInstance(tile_grid, ImageTileGrid)
        self.assertEqual((180, 180), tile_grid.tile_size)
        self.assertEqual(3, tile_grid.num_levels)

    def test_level_datasets(self):
        ds = _get_test_dataset()
        ml_ds = BaseMultiLevelDataset(ds)

        ds0 = ml_ds.get_dataset(0)
        self.assertIsNot(ds, ds0)
        self.assertEqual({'time': 14, 'lat': 720, 'lon': 1440, 'bnds': 2},
                         ds0.dims)

        ds1 = ml_ds.get_dataset(1)
        self.assertIsNot(ds, ds1)
        self.assertEqual({'time': 14, 'lat': 360, 'lon': 720},
                         ds1.dims)

        ds2 = ml_ds.get_dataset(2)
        self.assertIsNot(ds, ds2)
        self.assertEqual({'time': 14, 'lat': 180, 'lon': 360},
                         ds2.dims)

        self.assertEqual([ds0, ds1, ds2], ml_ds.datasets)

        ml_ds.close()

    def test_arg_validation(self):
        ds = _get_test_dataset()
        with self.assertRaises(TypeError):
            # noinspection PyTypeChecker
            BaseMultiLevelDataset('test.levels')
        with self.assertRaises(TypeError):
            # noinspection PyTypeChecker
            BaseMultiLevelDataset(ds, tile_grid=512)
        with self.assertRaises(TypeError):
            # noinspection PyTypeChecker
            BaseMultiLevelDataset(ds, grid_mapping='crs84')


class ComputedMultiLevelDatasetTest(unittest.TestCase):
    def test_it(self):
        ds = _get_test_dataset()

        ml_ds1 = BaseMultiLevelDataset(ds)

        def input_ml_dataset_getter(ds_id):
            if ds_id == "ml_ds1":
                return ml_ds1
            self.fail(f"unexpected ds_id={ds_id!r}")

        ml_ds2 = ComputedMultiLevelDataset(
            os.path.join(os.path.dirname(__file__),
                         "..", "webapi", "res", "test", "script.py"),
            "compute_dataset",
            ["ml_ds1"],
            input_ml_dataset_getter,
            input_parameters=dict(period='1W'),
            ds_id="ml_ds2"
        )
        self.assertEqual(3, ml_ds2.num_levels)
        self.assertIsInstance(ml_ds2.tile_grid, ImageTileGrid)
        self.assertEqual((180, 180), ml_ds2.tile_grid.tile_size)
        self.assertEqual(3, ml_ds2.tile_grid.num_levels)

        ds0 = ml_ds2.get_dataset(0)
        self.assertEqual({'time': 3, 'lat': 720, 'lon': 1440, 'bnds': 2}, ds0.dims)

        ds1 = ml_ds2.get_dataset(1)
        self.assertEqual({'time': 3, 'lat': 360, 'lon': 720}, ds1.dims)

        ds2 = ml_ds2.get_dataset(2)
        self.assertEqual({'time': 3, 'lat': 180, 'lon': 360}, ds2.dims)

        self.assertEqual([ds0, ds1, ds2], ml_ds2.datasets)

        ml_ds1.close()
        ml_ds2.close()


def _get_test_dataset(var_names=('noise',)):
    w = 1440
    h = 720
    p = 14

    x1 = -180.
    y1 = -90.
    x2 = +180.
    y2 = +90.
    dx = (x2 - x1) / w
    dy = (y2 - y1) / h

    data_vars = {var_name: (("time", "lat", "lon"), np.random.rand(p, h, w)) for var_name in var_names}

    lat_bnds = np.array(list(zip(np.linspace(y2, y1 + dy, num=h),
                                 np.linspace(y2 - dy, y1, num=h))))
    lon_bnds = np.array(list(zip(np.linspace(x1, x2 - dx, num=w),
                                 np.linspace(x1 + x2, x2, num=w))))
    coords = dict(time=(("time",),
                        np.array(pd.date_range(start="2019-01-01T12:00:00Z", periods=p, freq="1D"),
                                 dtype="datetime64[ns]")),
                  lat=(("lat",),
                       np.linspace(y2 - .5 * dy, y1 + .5 * dy, num=h)),
                  lon=(("lon",),
                       np.linspace(x1 + .5 * dx, x2 - .5 * dx, num=w)),
                  lat_bnds=(("lat", "bnds"), lat_bnds),
                  lon_bnds=(("lon", "bnds"), lon_bnds))

    return xr.Dataset(coords=coords, data_vars=data_vars).chunk(chunks=dict(lat=180, lon=180))


class ObjectStorageMultiLevelDatasetTest(S3Test):
    def test_s3_zarr(self):
        # if this test fails on travis with
        # E           IndexError: pop from an empty deque
        # then it most likly might be due to this issue: https://github.com/pydata/xarray/issues/4478
        # The issue has been fixed, but dask/s3fs does not have a new release including the fix yet

        self._write_test_cube()

        ml_ds_from_object_storage = open_ml_dataset_from_object_storage(
            'xcube-test-cube/cube-1-250-250.zarr',
            s3_kwargs=dict(key='test_fake_id', secret='test_fake_secret'),
            s3_client_kwargs=dict(endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        )
        self.assertIsNotNone(ml_ds_from_object_storage)
        self.assertIn('conc_chl', ml_ds_from_object_storage.base_dataset.variables)
        self.assertEqual((2, 200, 400), ml_ds_from_object_storage.base_dataset.conc_chl.shape)

    @classmethod
    def _write_test_cube(cls):
        s3_kwargs = dict(key='test_fake_id', secret='test_fake_secret')
        s3_client_kwargs = dict(endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        s3 = s3fs.S3FileSystem(**s3_kwargs, client_kwargs=s3_client_kwargs)
        # Create bucket 'xcube-test', so it exists before we write a test cube
        s3.mkdir('xcube-test-cube')

        # Create a test cube with just one variable "conc_chl"
        zarr_path = os.path.join(os.path.dirname(__file__), '../../examples/serve/demo/cube-1-250-250.zarr')
        dataset = xr.open_zarr(zarr_path)
        dataset = cls._make_subset(dataset)

        # Write test cube
        write_dataset(dataset,
                      'xcube-test-cube/cube-1-250-250.zarr',
                      s3_kwargs=s3_kwargs,
                      s3_client_kwargs=s3_client_kwargs)

    def test_s3_levels(self):
        self._write_test_cube_pyramid()

        s3 = s3fs.S3FileSystem(key='test_fake_id',
                               secret='test_fake_secret',
                               client_kwargs=dict(endpoint_url=MOTO_SERVER_ENDPOINT_URL))
        ml_dataset = ObjectStorageMultiLevelDataset(s3,
                                                    "xcube-test/cube-1-250-250.levels",
                                                    chunk_cache_capacity=1000 * 1000 * 1000)
        self.assertIsNotNone(ml_dataset)
        self.assertEqual(2, ml_dataset.num_levels)
        self.assertEqual((100, 100), ml_dataset.tile_grid.tile_size)
        self.assertEqual((2, 1), ml_dataset.tile_grid.get_num_tiles(0))
        self.assertEqual(800000000, ml_dataset.get_chunk_cache_capacity(0))
        self.assertEqual(200000000, ml_dataset.get_chunk_cache_capacity(1))

    @classmethod
    def _write_test_cube_pyramid(cls):
        s3_kwargs = dict(key='test_fake_id', secret='test_fake_secret')
        s3_client_kwargs = dict(endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        s3 = s3fs.S3FileSystem(**s3_kwargs, client_kwargs=s3_client_kwargs)
        # Create bucket 'xcube-test', so it exists before we write a test pyramid
        s3.mkdir('xcube-test')

        # Create a test cube pyramid with just one variable "conc_chl"
        zarr_path = os.path.join(os.path.dirname(__file__), '../../examples/serve/demo/cube-1-250-250.zarr')
        base_dataset = xr.open_zarr(zarr_path)
        base_dataset = cls._make_subset(base_dataset)
        ml_dataset = BaseMultiLevelDataset(base_dataset)

        # Write test cube pyramid
        write_levels(ml_dataset,
                     'xcube-test/cube-1-250-250.levels',
                     s3_kwargs=s3_kwargs,
                     s3_client_kwargs=s3_client_kwargs)

    @classmethod
    def _make_subset(cls, base_dataset):
        base_dataset = base_dataset.drop_vars(['c2rcc_flags', 'conc_tsm', 'kd489', 'quality_flags'])
        geometry = (1.0, 51.0, 2.0, 51.5)
        base_dataset = clip_dataset_by_geometry(base_dataset, geometry=geometry)
        base_dataset = base_dataset.dropna('time', how='all')
        # somehow clip_dataset_by_geometry() does not unify chunks and encoding, so it needs to be done manually:
        for var in base_dataset.variables:
            if var not in base_dataset.coords:
                if base_dataset[var].encoding['chunks'] != (1, 100, 100):
                    base_dataset[var].encoding['chunks'] = (1, 100, 100)
                base_dataset[var] = base_dataset[var].chunk({'time': 1, 'lat': 100, 'lon': 100})
        base_dataset['lat'] = base_dataset['lat'].chunk({'lat': 100})
        base_dataset.lat.encoding['chunks'] = (100,)
        base_dataset['lon'] = base_dataset['lon'].chunk({'lon': 100})
        base_dataset.lon.encoding['chunks'] = (100,)
        base_dataset['lat_bnds'] = base_dataset['lat_bnds'].chunk((100, 2))
        base_dataset.lat_bnds.encoding['chunks'] = (100, 2)
        base_dataset['lon_bnds'] = base_dataset['lon_bnds'].chunk((100, 2))
        base_dataset.lon_bnds.encoding['chunks'] = (100, 2)
        return base_dataset


# test/core/test_new.py

import unittest

import numpy as np

from xcube.core.new import new_cube


class NewCubeTest(unittest.TestCase):

    def test_new_cube_with_bounds(self):
        cube = new_cube()
        self.assertEqual({'lon': 360, 'lat': 180, 'time': 5, 'bnds': 2}, cube.dims)
        self.assertEqual(-179.5, float(cube.lon[0]))
        self.assertEqual(179.5, float(cube.lon[-1]))
        self.assertEqual(-89.5, float(cube.lat[0]))
        self.assertEqual(89.5, float(cube.lat[-1]))
        self.assertEqual(-180., float(cube.lon_bnds[0, 0]))
        self.assertEqual(-179., float(cube.lon_bnds[0, 1]))
        self.assertEqual(179., float(cube.lon_bnds[-1, 0]))
        self.assertEqual(180., float(cube.lon_bnds[-1, 1]))
        self.assertEqual(-90., float(cube.lat_bnds[0, 0]))
        self.assertEqual(-89., float(cube.lat_bnds[0, 1]))
        self.assertEqual(89., float(cube.lat_bnds[-1, 0]))
        self.assertEqual(90., float(cube.lat_bnds[-1, 1]))

    def test_new_cube_without_bounds(self):
        cube = new_cube(drop_bounds=True)
        self.assertEqual({'lon': 360, 'lat': 180, 'time': 5}, cube.dims)

    def test_new_cube_with_const_vars(self):
        cube = new_cube(variables=dict(sst=274.4, chl=10.31))
        self.assertIn('sst', cube)
        self.assertIn('chl', cube)
        import numpy as np
        np.testing.assert_almost_equal(cube.sst.values, np.full((5, 180, 360), 274.4))
        np.testing.assert_almost_equal(cube.chl.values, np.full((5, 180, 360), 10.31))

    def test_new_cube_with_func_vars(self):
        def sst_func(t, y, x):
            return 274.4 + t * 0.5

        def chl_func(t, y, x):
            return 10.31 + y * 0.1

        def aot_func(t, y, x):
            return 0.88 + x * 0.05

        cube = new_cube(width=16, height=8, variables=dict(sst=sst_func, chl=chl_func, aot=aot_func))

        self.assertIn('sst', cube)
        self.assertIn('chl', cube)
        self.assertIn('aot', cube)

        np.testing.assert_almost_equal(cube.sst[0, :, :].values, np.full((8, 16), 274.4))
        np.testing.assert_almost_equal(cube.chl[:, 0, :].values, np.full((5, 16), 10.31))
        np.testing.assert_almost_equal(cube.aot[:, :, 0].values, np.full((5, 8), 0.88))

        np.testing.assert_almost_equal(cube.sst.isel(lon=3, lat=2).values,
                                       np.array([274.4, 274.9, 275.4, 275.9, 276.4]))
        np.testing.assert_almost_equal(cube.chl.isel(time=3, lon=4).values,
                                       np.array([10.31, 10.41, 10.51, 10.61, 10.71, 10.81, 10.91, 11.01]))
        np.testing.assert_almost_equal(cube.aot.isel(time=1, lat=2).values,
                                       np.array([0.88, 0.93, 0.98, 1.03, 1.08, 1.13, 1.18, 1.23,
                                                 1.28, 1.33, 1.38, 1.43, 1.48, 1.53, 1.58, 1.63]))


# test/core/test_normalize.py

"""
Test for the normalization operation
"""

from datetime import datetime
from unittest import TestCase

import numpy as np
import pandas as pd
import pyproj
import xarray as xr
from jdcal import gcal2jd
from numpy.testing import assert_array_almost_equal

from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.normalize import DatasetIsNotACubeError
from xcube.core.normalize import adjust_spatial_attrs
from xcube.core.normalize import decode_cube
from xcube.core.normalize import encode_cube
from xcube.core.normalize import normalize_coord_vars
from xcube.core.normalize import normalize_dataset
from xcube.core.normalize import normalize_missing_time


# noinspection PyPep8Naming
def assertDatasetEqual(expected, actual):
    # this method is functionally equivalent to
    # `assert expected == actual`, but it
    # checks each aspect of equality separately for easier debugging
    assert expected.equals(actual), (expected, actual)


class DecodeCubeTest(TestCase):
    def test_cube_stays_cube(self):
        dataset = new_cube(variables=dict(a=1, b=2, c=3))
        cube, grid_mapping, rest = decode_cube(dataset)
        self.assertIs(dataset, cube)
        self.assertIsInstance(grid_mapping, GridMapping)
        self.assertTrue(grid_mapping.crs.is_geographic)
        self.assertIsInstance(rest, xr.Dataset)
        self.assertEqual(set(), set(rest.data_vars))

    def test_no_cube_vars_are_dropped(self):
        dataset = new_cube(variables=dict(a=1, b=2, c=3))
        dataset = dataset.assign(
            d=xr.DataArray([8, 9, 10], dims='level'),
            crs=xr.DataArray(0, attrs=pyproj.CRS.from_string('CRS84').to_cf()),
        )
        self.assertEqual({'a', 'b', 'c', 'd', 'crs'}, set(dataset.data_vars))
        cube, grid_mapping, rest = decode_cube(dataset)
        self.assertIsInstance(cube, xr.Dataset)
        self.assertIsInstance(grid_mapping, GridMapping)
        self.assertEqual({'a', 'b', 'c'}, set(cube.data_vars))
        self.assertEqual(pyproj.CRS.from_string('CRS84'), grid_mapping.crs)
        self.assertIsInstance(rest, xr.Dataset)
        self.assertEqual({'d', 'crs'}, set(rest.data_vars))

    def test_encode_is_inverse(self):
        dataset = new_cube(variables=dict(a=1, b=2, c=3),
                           x_name='x', y_name='y')
        dataset = dataset.assign(
            d=xr.DataArray([8, 9, 10], dims='level'),
            crs=xr.DataArray(0, attrs=pyproj.CRS.from_string('CRS84').to_cf()),
        )
        cube, grid_mapping, rest = decode_cube(dataset)
        dataset2 = encode_cube(cube, grid_mapping, rest)
        self.assertEqual(set(dataset.data_vars), set(dataset2.data_vars))
        self.assertIn('crs', dataset2.data_vars)

    def test_no_cube_vars_found(self):
        dataset = new_cube()
        self.assertEqual(set(), set(dataset.data_vars))
        with self.assertRaises(DatasetIsNotACubeError) as cm:
            decode_cube(dataset, force_non_empty=True)
        self.assertEqual("No variables found with dimensions"
                         " ('time', [...] 'lat', 'lon')"
                         " or dimension sizes too small",
                         f'{cm.exception}')

    def test_no_grid_mapping(self):
        dataset = xr.Dataset(dict(a=[1, 2, 3], b=0.5))
        with self.assertRaises(DatasetIsNotACubeError) as cm:
            decode_cube(dataset)
        self.assertEqual("Failed to detect grid mapping:"
                         " cannot find any grid mapping in dataset",
                         f'{cm.exception}')

    def test_grid_mapping_not_geographic(self):
        dataset = new_cube(x_name='x', y_name='y',
                           variables=dict(a=0.5), crs='epsg:25832')
        with self.assertRaises(DatasetIsNotACubeError) as cm:
            decode_cube(dataset, force_geographic=True)
        self.assertEqual("Grid mapping must use geographic CRS,"
                         " but was 'ETRS89 / UTM zone 32N'",
                         f'{cm.exception}')


class EncodeCubeTest(TestCase):
    def test_geographical_crs(self):
        cube = new_cube(variables=dict(a=1, b=2, c=3))
        gm = GridMapping.from_dataset(cube)

        dataset = encode_cube(cube, gm)
        self.assertIs(cube, dataset)

        dataset = encode_cube(cube, gm,
                              xr.Dataset(dict(d=True)))
        self.assertIsInstance(dataset, xr.Dataset)
        self.assertEqual({'a', 'b', 'c', 'd'}, set(dataset.data_vars))

    def test_non_geographical_crs(self):
        cube = new_cube(x_name='x',
                        y_name='y',
                        crs='epsg:25832',
                        variables=dict(a=1, b=2, c=3))
        gm = GridMapping.from_dataset(cube)
        dataset = encode_cube(cube,
                              gm,
                              xr.Dataset(dict(d=True)))
        self.assertIsInstance(dataset, xr.Dataset)
        self.assertEqual({'a', 'b', 'c', 'd', 'crs'}, set(dataset.data_vars))


class TestNormalize(TestCase):

    def test_normalize_zonal_lat_lon(self):
        resolution = 10
        lat_size = 3
        lat_coords = np.arange(0, 30, resolution)
        lon_coords = [i + 5. for i in np.arange(-180.0, 180.0, resolution)]
        lon_size = len(lon_coords)
        one_more_dim_size = 2
        one_more_dim_coords = np.random.random(2)

        var_values_1_1d = xr.DataArray(np.random.random(lat_size),
                                       coords=[('latitude_centers', lat_coords)],
                                       dims=['latitude_centers'],
                                       attrs=dict(chunk_sizes=[lat_size],
                                                  dimensions=['latitude_centers']))
        var_values_1_1d.encoding = {'chunks': (lat_size,)}
        var_values_1_2d = xr.DataArray(np.array([var_values_1_1d.values for _ in lon_coords]).T,
                                       coords={'lat': lat_coords, 'lon': lon_coords},
                                       dims=['lat', 'lon'],
                                       attrs=dict(chunk_sizes=[lat_size, lon_size],
                                                  dimensions=['lat', 'lon']))
        var_values_1_2d.encoding = {'chunks': (lat_size, lon_size)}
        var_values_2_2d = xr.DataArray(np.random.random(lat_size * one_more_dim_size).
                                       reshape(lat_size, one_more_dim_size),
                                       coords={'latitude_centers': lat_coords,
                                               'one_more_dim': one_more_dim_coords},
                                       dims=['latitude_centers', 'one_more_dim'],
                                       attrs=dict(chunk_sizes=[lat_size, one_more_dim_size],
                                                  dimensions=['latitude_centers', 'one_more_dim']))
        var_values_2_2d.encoding = {'chunks': (lat_size, one_more_dim_size)}
        var_values_2_3d = xr.DataArray(np.array([var_values_2_2d.values for _ in lon_coords]).T,
                                       coords={'one_more_dim': one_more_dim_coords,
                                               'lat': lat_coords,
                                               'lon': lon_coords, },
                                       dims=['one_more_dim', 'lat', 'lon', ],
                                       attrs=dict(chunk_sizes=[one_more_dim_size,
                                                               lat_size,
                                                               lon_size],
                                                  dimensions=['one_more_dim', 'lat', 'lon']))
        var_values_2_3d.encoding = {'chunks': (one_more_dim_size, lat_size, lon_size)}

        dataset = xr.Dataset({'first': var_values_1_1d, 'second': var_values_2_2d})
        expected = xr.Dataset({'first': var_values_1_2d, 'second': var_values_2_3d})
        expected = expected.assign_coords(
            lon_bnds=xr.DataArray([[i - (resolution / 2), i + (resolution / 2)] for i in expected.lon.values],
                                  dims=['lon', 'bnds']))
        expected = expected.assign_coords(
            lat_bnds=xr.DataArray([[i - (resolution / 2), i + (resolution / 2)] for i in expected.lat.values],
                                  dims=['lat', 'bnds']))
        actual = normalize_dataset(dataset)

        xr.testing.assert_equal(actual, expected)
        self.assertEqual(actual.first.chunk_sizes, expected.first.chunk_sizes)
        self.assertEqual(actual.second.chunk_sizes, expected.second.chunk_sizes)

    def test_normalize_lon_lat_2d(self):
        """
        Test nominal execution
        """
        dims = ('time', 'y', 'x')
        attrs = {'valid_min': 0., 'valid_max': 1.}

        t_size = 2
        y_size = 3
        x_size = 4

        a_data = np.random.random_sample((t_size, y_size, x_size))
        b_data = np.random.random_sample((t_size, y_size, x_size))
        time_data = [1, 2]
        lat_data = [[10., 10., 10., 10.],
                    [20., 20., 20., 20.],
                    [30., 30., 30., 30.]]
        lon_data = [[-10., 0., 10., 20.],
                    [-10., 0., 10., 20.],
                    [-10., 0., 10., 20.]]
        dataset = xr.Dataset({'a': (dims, a_data, attrs),
                              'b': (dims, b_data, attrs)
                              },
                             {'time': (('time',), time_data),
                              'lat': (('y', 'x'), lat_data),
                              'lon': (('y', 'x'), lon_data)
                              },
                             {'geospatial_lon_min': -15.,
                              'geospatial_lon_max': 25.,
                              'geospatial_lat_min': 5.,
                              'geospatial_lat_max': 35.
                              }
                             )

        new_dims = ('time', 'lat', 'lon')
        expected = xr.Dataset({'a': (new_dims, a_data, attrs),
                               'b': (new_dims, b_data, attrs)},
                              {'time': (('time',), time_data),
                               'lat': (('lat',), [10., 20., 30.]),
                               'lon': (('lon',), [-10., 0., 10., 20.]),
                               },
                              {'geospatial_lon_min': -15.,
                               'geospatial_lon_max': 25.,
                               'geospatial_lat_min': 5.,
                               'geospatial_lat_max': 35.})

        actual = normalize_dataset(dataset)
        xr.testing.assert_equal(actual, expected)

    def test_normalize_lon_lat(self):
        """
        Test nominal execution
        """
        dataset = xr.Dataset({'first': (['latitude',
                                         'longitude'], [[1, 2, 3],
                                                        [2, 3, 4]])})
        expected = xr.Dataset({'first': (['lat', 'lon'], [[1, 2, 3],
                                                          [2, 3, 4]])})
        actual = normalize_dataset(dataset)
        assertDatasetEqual(actual, expected)

        dataset = xr.Dataset({'first': (['lat', 'long'], [[1, 2, 3],
                                                          [2, 3, 4]])})
        expected = xr.Dataset({'first': (['lat', 'lon'], [[1, 2, 3],
                                                          [2, 3, 4]])})
        actual = normalize_dataset(dataset)
        assertDatasetEqual(actual, expected)

        dataset = xr.Dataset({'first': (['latitude',
                                         'spacetime'], [[1, 2, 3],
                                                        [2, 3, 4]])})
        expected = xr.Dataset({'first': (['lat', 'spacetime'], [[1, 2, 3],
                                                                [2, 3, 4]])})
        actual = normalize_dataset(dataset)
        assertDatasetEqual(actual, expected)

        dataset = xr.Dataset({'first': (['zef', 'spacetime'], [[1, 2, 3],
                                                               [2, 3, 4]])})
        expected = xr.Dataset({'first': (['zef', 'spacetime'], [[1, 2, 3],
                                                                [2, 3, 4]])})
        actual = normalize_dataset(dataset)
        assertDatasetEqual(actual, expected)

    def test_normalize_does_not_reorder_increasing_lat(self):
        first = np.zeros([3, 45, 90])
        first[0, :, :] = np.eye(45, 90)
        ds = xr.Dataset({
            'first': (['time', 'lat', 'lon'], first),
            'second': (['time', 'lat', 'lon'], np.zeros([3, 45, 90])),
            'lat': np.linspace(-88, 88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 4)]}).chunk(
            chunks={'time': 1})

        actual = normalize_dataset(ds)
        xr.testing.assert_equal(actual, ds)

    def test_normalize_with_missing_time_dim(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101',
                               'time_coverage_end': '20121231'})
        norm_ds = normalize_dataset(ds)
        self.assertIsNot(norm_ds, ds)
        self.assertEqual(len(norm_ds.coords), 4)
        self.assertIn('lon', norm_ds.coords)
        self.assertIn('lat', norm_ds.coords)
        self.assertIn('time', norm_ds.coords)
        self.assertIn('time_bnds', norm_ds.coords)

        self.assertEqual(norm_ds.first.shape, (1, 90, 180))
        self.assertEqual(norm_ds.second.shape, (1, 90, 180))
        self.assertEqual(norm_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2012-07-01T12:00:00')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][0], xr.DataArray(pd.to_datetime('2012-01-01')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][1], xr.DataArray(pd.to_datetime('2012-12-31')))

    def test_normalize_with_time_called_t(self):
        ds = xr.Dataset({'first': (['time', 'lat', 'lon'], np.zeros([4, 90, 180])),
                         'second': (['time', 'lat', 'lon'], np.zeros([4, 90, 180])),
                         't': ('time', np.array(['2005-07-02T00:00:00.000000000',
                                                 '2006-07-02T12:00:00.000000000',
                                                 '2007-07-03T00:00:00.000000000',
                                                 '2008-07-02T00:00:00.000000000'], dtype='datetime64[ns]'))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '2005-01-17',
                               'time_coverage_end': '2008-08-17'})
        norm_ds = normalize_dataset(ds)
        self.assertIsNot(norm_ds, ds)
        self.assertEqual(len(norm_ds.coords), 3)
        self.assertIn('lon', norm_ds.coords)
        self.assertIn('lat', norm_ds.coords)
        self.assertIn('time', norm_ds.coords)

        self.assertEqual(norm_ds.first.shape, (4, 90, 180))
        self.assertEqual(norm_ds.second.shape, (4, 90, 180))
        self.assertEqual(norm_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2005-07-02T00:00')))

    def test_normalize_julian_day(self):
        """
        Test Julian Day -> Datetime conversion
        """
        tuples = [gcal2jd(2000, x, 1) for x in range(1, 13)]

        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([88, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([88, 90, 12])),
            'lat': np.linspace(-88, 45, 88),
            'lon': np.linspace(-178, 178, 90),
            'time': [x[0] + x[1] for x in tuples]})
        ds.time.attrs['long_name'] = 'time in julian days'

        expected = xr.Dataset({
            'first': (['time', 'lat', 'lon'], np.zeros([12, 88, 90])),
            'second': (['time', 'lat', 'lon'], np.zeros([12, 88, 90])),
            'lat': np.linspace(-88, 45, 88),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})
        expected.time.attrs['long_name'] = 'time'

        actual = normalize_dataset(ds)

        assertDatasetEqual(actual, expected)


class AdjustSpatialTest(TestCase):
    def test_nominal(self):
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'lat': np.linspace(-88, 88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})

        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        ds1 = adjust_spatial_attrs(ds)

        # Make sure original dataset is not altered
        with self.assertRaises(KeyError):
            # noinspection PyStatementEffect
            ds.attrs['geospatial_lat_min']

        # Make sure expected values are in the new dataset
        self.assertEqual(ds1.attrs['geospatial_lat_min'], -90)
        self.assertEqual(ds1.attrs['geospatial_lat_max'], 90)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds1.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_lon_min'], -180)
        self.assertEqual(ds1.attrs['geospatial_lon_max'], 180)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((-180.0 -90.0, -180.0 90.0, 180.0 90.0,'
                         ' 180.0 -90.0, -180.0 -90.0))')

        # Test existing attributes update
        lon_min, lat_min, lon_max, lat_max = -20, -40, 60, 40
        indexers = {'lon': slice(lon_min, lon_max),
                    'lat': slice(lat_min, lat_max)}
        ds2 = ds1.sel(**indexers)
        ds2 = adjust_spatial_attrs(ds2)

        self.assertEqual(ds2.attrs['geospatial_lat_min'], -42)
        self.assertEqual(ds2.attrs['geospatial_lat_max'], 42)
        self.assertEqual(ds2.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds2.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_lon_min'], -20)
        self.assertEqual(ds2.attrs['geospatial_lon_max'], 60)
        self.assertEqual(ds2.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds2.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_bounds'],
                         'POLYGON((-20.0 -42.0, -20.0 42.0, 60.0 42.0, 60.0'
                         ' -42.0, -20.0 -42.0))')

    def test_nominal_inverted(self):
        # Inverted lat
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'lat': np.linspace(88, -88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})

        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        ds1 = adjust_spatial_attrs(ds)

        # Make sure original dataset is not altered
        with self.assertRaises(KeyError):
            # noinspection PyStatementEffect
            ds.attrs['geospatial_lat_min']

        # Make sure expected values are in the new dataset
        self.assertEqual(ds1.attrs['geospatial_lat_min'], -90)
        self.assertEqual(ds1.attrs['geospatial_lat_max'], 90)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds1.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_lon_min'], -180)
        self.assertEqual(ds1.attrs['geospatial_lon_max'], 180)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((-180.0 -90.0, -180.0 90.0, 180.0 90.0,'
                         ' 180.0 -90.0, -180.0 -90.0))')

        # Test existing attributes update
        lon_min, lat_min, lon_max, lat_max = -20, -40, 60, 40
        indexers = {'lon': slice(lon_min, lon_max),
                    'lat': slice(lat_max, lat_min)}
        ds2 = ds1.sel(**indexers)
        ds2 = adjust_spatial_attrs(ds2)

        self.assertEqual(ds2.attrs['geospatial_lat_min'], -42)
        self.assertEqual(ds2.attrs['geospatial_lat_max'], 42)
        self.assertEqual(ds2.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds2.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_lon_min'], -20)
        self.assertEqual(ds2.attrs['geospatial_lon_max'], 60)
        self.assertEqual(ds2.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds2.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_bounds'],
                         'POLYGON((-20.0 -42.0, -20.0 42.0, 60.0 42.0, 60.0'
                         ' -42.0, -20.0 -42.0))')

    def test_bnds(self):
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'lat': np.linspace(-88, 88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})

        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        lat_bnds = np.empty([len(ds.lat), 2])
        lon_bnds = np.empty([len(ds.lon), 2])
        ds['nv'] = [0, 1]

        lat_bnds[:, 0] = ds.lat.values - 2
        lat_bnds[:, 1] = ds.lat.values + 2
        lon_bnds[:, 0] = ds.lon.values - 2
        lon_bnds[:, 1] = ds.lon.values + 2

        ds['lat_bnds'] = (['lat', 'nv'], lat_bnds)
        ds['lon_bnds'] = (['lon', 'nv'], lon_bnds)

        ds.lat.attrs['bounds'] = 'lat_bnds'
        ds.lon.attrs['bounds'] = 'lon_bnds'

        ds1 = adjust_spatial_attrs(ds)

        # Make sure original dataset is not altered
        with self.assertRaises(KeyError):
            # noinspection PyStatementEffect
            ds.attrs['geospatial_lat_min']

        # Make sure expected values are in the new dataset
        self.assertEqual(ds1.attrs['geospatial_lat_min'], -90)
        self.assertEqual(ds1.attrs['geospatial_lat_max'], 90)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds1.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_lon_min'], -180)
        self.assertEqual(ds1.attrs['geospatial_lon_max'], 180)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((-180.0 -90.0, -180.0 90.0, 180.0 90.0,'
                         ' 180.0 -90.0, -180.0 -90.0))')

        # Test existing attributes update
        lon_min, lat_min, lon_max, lat_max = -20, -40, 60, 40
        indexers = {'lon': slice(lon_min, lon_max),
                    'lat': slice(lat_min, lat_max)}
        ds2 = ds1.sel(**indexers)
        ds2 = adjust_spatial_attrs(ds2)

        self.assertEqual(ds2.attrs['geospatial_lat_min'], -42)
        self.assertEqual(ds2.attrs['geospatial_lat_max'], 42)
        self.assertEqual(ds2.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds2.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_lon_min'], -20)
        self.assertEqual(ds2.attrs['geospatial_lon_max'], 60)
        self.assertEqual(ds2.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds2.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_bounds'],
                         'POLYGON((-20.0 -42.0, -20.0 42.0, 60.0 42.0, 60.0'
                         ' -42.0, -20.0 -42.0))')

    def test_bnds_inverted(self):
        # Inverted lat
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'lat': np.linspace(88, -88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})

        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        lat_bnds = np.empty([len(ds.lat), 2])
        lon_bnds = np.empty([len(ds.lon), 2])
        ds['nv'] = [0, 1]

        lat_bnds[:, 0] = ds.lat.values + 2
        lat_bnds[:, 1] = ds.lat.values - 2
        lon_bnds[:, 0] = ds.lon.values - 2
        lon_bnds[:, 1] = ds.lon.values + 2

        ds['lat_bnds'] = (['lat', 'nv'], lat_bnds)
        ds['lon_bnds'] = (['lon', 'nv'], lon_bnds)

        ds.lat.attrs['bounds'] = 'lat_bnds'
        ds.lon.attrs['bounds'] = 'lon_bnds'

        ds1 = adjust_spatial_attrs(ds)

        # Make sure original dataset is not altered
        with self.assertRaises(KeyError):
            # noinspection PyStatementEffect
            ds.attrs['geospatial_lat_min']

        # Make sure expected values are in the new dataset
        self.assertEqual(ds1.attrs['geospatial_lat_min'], -90)
        self.assertEqual(ds1.attrs['geospatial_lat_max'], 90)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds1.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_lon_min'], -180)
        self.assertEqual(ds1.attrs['geospatial_lon_max'], 180)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((-180.0 -90.0, -180.0 90.0, 180.0 90.0,'
                         ' 180.0 -90.0, -180.0 -90.0))')

        # Test existing attributes update
        lon_min, lat_min, lon_max, lat_max = -20, -40, 60, 40
        indexers = {'lon': slice(lon_min, lon_max),
                    'lat': slice(lat_max, lat_min)}
        ds2 = ds1.sel(**indexers)
        ds2 = adjust_spatial_attrs(ds2)

        self.assertEqual(ds2.attrs['geospatial_lat_min'], -42)
        self.assertEqual(ds2.attrs['geospatial_lat_max'], 42)
        self.assertEqual(ds2.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds2.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_lon_min'], -20)
        self.assertEqual(ds2.attrs['geospatial_lon_max'], 60)
        self.assertEqual(ds2.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds2.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_bounds'],
                         'POLYGON((-20.0 -42.0, -20.0 42.0, 60.0 42.0, 60.0 -42.0, -20.0 -42.0))')

    def test_once_cell_with_bnds(self):
        # Only one cell in lat/lon
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([1, 1, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([1, 1, 12])),
            'lat': np.array([52.5]),
            'lon': np.array([11.5]),
            'lat_bnds': (['lat', 'bnds'], np.array([[52.4, 52.6]])),
            'lon_bnds': (['lon', 'bnds'], np.array([[11.4, 11.6]])),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})
        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        ds1 = adjust_spatial_attrs(ds)
        self.assertAlmostEqual(ds1.attrs['geospatial_lat_resolution'], 0.2)
        self.assertAlmostEqual(ds1.attrs['geospatial_lat_min'], 52.4)
        self.assertAlmostEqual(ds1.attrs['geospatial_lat_max'], 52.6)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertAlmostEqual(ds1.attrs['geospatial_lon_resolution'], 0.2)
        self.assertAlmostEqual(ds1.attrs['geospatial_lon_min'], 11.4)
        self.assertAlmostEqual(ds1.attrs['geospatial_lon_max'], 11.6)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((11.4 52.4, 11.4 52.6, 11.6 52.6, 11.6 52.4, 11.4 52.4))')

    def test_once_cell_without_bnds(self):
        # Only one cell in lat/lon
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([1, 1, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([1, 1, 12])),
            'lat': np.array([52.5]),
            'lon': np.array([11.5]),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})
        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        ds2 = adjust_spatial_attrs(ds)
        # Datasets should be the same --> not modified
        self.assertIs(ds2, ds)


class NormalizeCoordVarsTest(TestCase):

    def test_ds_with_potential_coords(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180])),
                         'lat_bnds': (['lat', 'bnds'], np.zeros([90, 2])),
                         'lon_bnds': (['lon', 'bnds'], np.zeros([180, 2]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)})

        new_ds = normalize_coord_vars(ds)

        self.assertIsNot(ds, new_ds)
        self.assertEqual(len(new_ds.coords), 4)
        self.assertIn('lon', new_ds.coords)
        self.assertIn('lat', new_ds.coords)
        self.assertIn('lat_bnds', new_ds.coords)
        self.assertIn('lon_bnds', new_ds.coords)

        self.assertEqual(len(new_ds.data_vars), 2)
        self.assertIn('first', new_ds.data_vars)
        self.assertIn('second', new_ds.data_vars)

    def test_ds_with_potential_coords_and_bounds(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180])),
                         'lat_bnds': (['lat', 'bnds'], np.zeros([90, 2])),
                         'lon_bnds': (['lon', 'bnds'], np.zeros([180, 2])),
                         'lat': (['lat'], np.linspace(-89.5, 89.5, 90)),
                         'lon': (['lon'], np.linspace(-179.5, 179.5, 180))})

        new_ds = normalize_coord_vars(ds)

        self.assertIsNot(ds, new_ds)
        self.assertEqual(len(new_ds.coords), 4)
        self.assertIn('lon', new_ds.coords)
        self.assertIn('lat', new_ds.coords)
        self.assertIn('lat_bnds', new_ds.coords)
        self.assertIn('lon_bnds', new_ds.coords)

        self.assertEqual(len(new_ds.data_vars), 2)
        self.assertIn('first', new_ds.data_vars)
        self.assertIn('second', new_ds.data_vars)

    def test_ds_with_no_potential_coords(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101'})
        new_ds = normalize_coord_vars(ds)
        self.assertIs(ds, new_ds)


class NormalizeMissingTimeTest(TestCase):
    def test_ds_without_time(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101',
                               'time_coverage_end': '20121231'})

        new_ds = normalize_missing_time(ds)

        self.assertIsNot(ds, new_ds)
        self.assertEqual(len(new_ds.coords), 4)
        self.assertIn('lon', new_ds.coords)
        self.assertIn('lat', new_ds.coords)
        self.assertIn('time', new_ds.coords)
        self.assertIn('time_bnds', new_ds.coords)

        self.assertEqual(new_ds.coords['time'].attrs.get('long_name'), 'time')
        self.assertEqual(new_ds.coords['time'].attrs.get('bounds'), 'time_bnds')

        self.assertEqual(new_ds.first.shape, (1, 90, 180))
        self.assertEqual(new_ds.second.shape, (1, 90, 180))
        self.assertEqual(new_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2012-07-01T12:00:00')))
        self.assertEqual(new_ds.coords['time'].attrs.get('long_name'), 'time')
        self.assertEqual(new_ds.coords['time'].attrs.get('bounds'), 'time_bnds')
        self.assertEqual(new_ds.coords['time_bnds'][0][0], xr.DataArray(pd.to_datetime('2012-01-01')))
        self.assertEqual(new_ds.coords['time_bnds'][0][1], xr.DataArray(pd.to_datetime('2012-12-31')))
        self.assertEqual(new_ds.coords['time_bnds'].attrs.get('long_name'), 'time')

    def test_ds_without_bounds(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101'})

        new_ds = normalize_missing_time(ds)

        self.assertIsNot(ds, new_ds)
        self.assertEqual(len(new_ds.coords), 3)
        self.assertIn('lon', new_ds.coords)
        self.assertIn('lat', new_ds.coords)
        self.assertIn('time', new_ds.coords)
        self.assertNotIn('time_bnds', new_ds.coords)

        self.assertEqual(new_ds.first.shape, (1, 90, 180))
        self.assertEqual(new_ds.second.shape, (1, 90, 180))
        self.assertEqual(new_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2012-01-01')))
        self.assertEqual(new_ds.coords['time'].attrs.get('long_name'), 'time')
        self.assertEqual(new_ds.coords['time'].attrs.get('bounds'), None)

    def test_ds_without_time_attrs(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)})

        new_ds = normalize_missing_time(ds)
        self.assertIs(ds, new_ds)

    def test_ds_with_cftime(self):
        time_data = xr.cftime_range(start='2010-01-01T00:00:00',
                                    periods=6,
                                    freq='D',
                                    calendar='gregorian').values
        ds = xr.Dataset({'first': (['time', 'lat', 'lon'], np.zeros([6, 90, 180])),
                         'second': (['time', 'lat', 'lon'], np.zeros([6, 90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180),
                                'time': time_data},
                        attrs={'time_coverage_start': '20120101',
                               'time_coverage_end': '20121231'})
        new_ds = normalize_missing_time(ds)
        self.assertIs(ds, new_ds)

    def test_normalize_with_missing_time_dim(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101',
                               'time_coverage_end': '20121231'})
        norm_ds = normalize_dataset(ds)
        self.assertIsNot(norm_ds, ds)
        self.assertEqual(len(norm_ds.coords), 4)
        self.assertIn('lon', norm_ds.coords)
        self.assertIn('lat', norm_ds.coords)
        self.assertIn('time', norm_ds.coords)
        self.assertIn('time_bnds', norm_ds.coords)

        self.assertEqual(norm_ds.first.shape, (1, 90, 180))
        self.assertEqual(norm_ds.second.shape, (1, 90, 180))
        self.assertEqual(norm_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2012-07-01T12:00:00')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][0], xr.DataArray(pd.to_datetime('2012-01-01')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][1], xr.DataArray(pd.to_datetime('2012-12-31')))

    def test_normalize_with_missing_time_dim_from_filename(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        )
        ds_encoding = dict(source='20150204_etfgz_20170309_dtsrgth')
        ds.encoding.update(ds_encoding)
        norm_ds = normalize_dataset(ds)
        self.assertIsNot(norm_ds, ds)
        self.assertEqual(len(norm_ds.coords), 4)
        self.assertIn('lon', norm_ds.coords)
        self.assertIn('lat', norm_ds.coords)
        self.assertIn('time', norm_ds.coords)
        self.assertIn('time_bnds', norm_ds.coords)

        self.assertEqual(norm_ds.first.shape, (1, 90, 180))
        self.assertEqual(norm_ds.second.shape, (1, 90, 180))
        self.assertEqual(norm_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2016-02-21T00:00:00')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][0], xr.DataArray(pd.to_datetime('2015-02-04')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][1], xr.DataArray(pd.to_datetime('2017-03-09')))


class Fix360Test(TestCase):

    def test_fix_360_lon(self):
        # The following simulates a strangely geo-coded soil moisture dataset we found
        lon_size = 360
        lat_size = 130
        time_size = 12
        ds = xr.Dataset({
            'first': (['time', 'lat', 'lon'],
                      np.random.random_sample([time_size, lat_size, lon_size])),
            'second': (['time', 'lat', 'lon'],
                       np.random.random_sample([time_size, lat_size,
                                                lon_size]))},
            coords={'lon': np.linspace(1., 360., lon_size),
                    'lat': np.linspace(-65., 64., lat_size),
                    'time': [datetime(2000, x, 1)
                             for x in range(1, time_size + 1)]},
            attrs=dict(geospatial_lon_min=0.,
                       geospatial_lon_max=360.,
                       geospatial_lat_min=-65.5,
                       geospatial_lat_max=+64.5,
                       geospatial_lon_resolution=1.,
                       geospatial_lat_resolution=1.))

        new_ds = normalize_dataset(ds)
        self.assertIsNot(ds, new_ds)
        self.assertEqual(ds.dims, new_ds.dims)
        self.assertEqual(ds.sizes, new_ds.sizes)
        assert_array_almost_equal(new_ds.lon, np.linspace(-179.5, 179.5, 360))
        assert_array_almost_equal(new_ds.lat, np.linspace(-65., 64., 130))
        assert_array_almost_equal(new_ds.first[..., :180], ds.first[..., 180:])
        assert_array_almost_equal(new_ds.first[..., 180:], ds.first[..., :180])
        assert_array_almost_equal(new_ds.second[..., :180],
                                  ds.second[..., 180:])
        assert_array_almost_equal(new_ds.second[..., 180:],
                                  ds.second[..., :180])
        self.assertEqual(-180., new_ds.attrs['geospatial_lon_min'])
        self.assertEqual(+180., new_ds.attrs['geospatial_lon_max'])
        self.assertEqual(-65.5, new_ds.attrs['geospatial_lat_min'])
        self.assertEqual(+64.5, new_ds.attrs['geospatial_lat_max'])
        self.assertEqual(1., new_ds.attrs['geospatial_lon_resolution'])
        self.assertEqual(1., new_ds.attrs['geospatial_lat_resolution'])


class NormalizeDimOrderTest(TestCase):
    """
    Test normalize_cci_sea_level operation
    """

    def test_no_change(self):
        """
        Test nominal operation
        """
        lon_size = 360
        lat_size = 130
        time_size = 12
        ds = xr.Dataset({
            'first': (['time', 'lat', 'lon'],
                      np.random.random_sample([time_size, lat_size, lon_size])),
            'second': (['time', 'lat', 'lon'],
                       np.random.random_sample([time_size, lat_size,
                                                lon_size]))},
            coords={'lon': np.linspace(-179.5, -179.5, lon_size),
                    'lat': np.linspace(-65., 64., lat_size),
                    'time': [datetime(2000, x, 1)
                             for x in range(1, time_size + 1)]})
        ds2 = normalize_dataset(ds)
        self.assertIs(ds2, ds)

    def test_nominal(self):
        """
        Test nominal operation
        """
        ds = self.new_cci_seal_level_ds()
        ds2 = normalize_dataset(ds)
        self.assertIsNot(ds2, ds)
        self.assertIn('ampl', ds2)
        self.assertIn('phase', ds2)
        self.assertIn('time', ds2.coords)
        self.assertIn('time_bnds', ds2.coords)
        self.assertNotIn('time_step', ds2.coords)
        self.assertEqual(['time', 'period', 'lat', 'lon'], list(ds2.ampl.dims))
        self.assertEqual(['time', 'period', 'lat', 'lon'], list(ds2.phase.dims))

    @staticmethod
    def new_cci_seal_level_ds():
        period_size = 2
        lon_size = 4
        lat_size = 2

        dataset = xr.Dataset(dict(ampl=(['lat', 'lon', 'period'], np.ones(shape=(lat_size, lon_size, period_size))),
                                  phase=(['lat', 'lon', 'period'], np.zeros(shape=(lat_size, lon_size, period_size)))),
                             coords=dict(lon=np.array([-135, -45., 45., 135.]), lat=np.array([-45., 45.]),
                                         time=pd.to_datetime(
                                             ['1993-01-15T00:00:00.000000000', '1993-02-15T00:00:00.000000000',
                                              '2015-11-15T00:00:00.000000000', '2015-12-15T00:00:00.000000000'])))

        dataset.coords['time'].encoding.update(units='days since 1950-01-01', dtype=np.dtype(np.float32))
        dataset.coords['time'].attrs.update(long_name='time', standard_name='time')

        dataset.attrs['time_coverage_start'] = '1993-01-01 00:00:00'
        dataset.attrs['time_coverage_end'] = '2015-12-31 23:59:59'

        return dataset


# test/core/test_optimize.py

import os
import os.path
import unittest
from typing import Set, Union, Sequence

from xcube.constants import FORMAT_NAME_ZARR
from xcube.core.chunk import chunk_dataset
from xcube.core.dsio import rimraf
from xcube.core.new import new_cube
from xcube.core.optimize import optimize_dataset

TEST_CUBE = chunk_dataset(new_cube(time_periods=3, variables=dict(A=0.5, B=-1.5)),
                          chunk_sizes=dict(time=1, lat=180, lon=360), format_name=FORMAT_NAME_ZARR)

INPUT_CUBE_PATH = 'test.zarr'
OUTPUT_CUBE_PATH = 'test_opt.zarr'
OUTPUT_CUBE_PATTERN = '{input}_opt.zarr'

INPUT_CUBE_FILE_SET = {
    '.zattrs', '.zgroup',  '.zmetadata',
    'A/.zarray', 'A/.zattrs', 'A/0.0.0', 'A/1.0.0', 'A/2.0.0',
    'B/.zarray', 'B/.zattrs', 'B/0.0.0', 'B/1.0.0', 'B/2.0.0',
    'lat/.zarray', 'lat/.zattrs', 'lat/0',
    'lat_bnds/.zarray', 'lat_bnds/.zattrs', 'lat_bnds/0.0',
    'lon/.zarray', 'lon/.zattrs', 'lon/0',
    'lon_bnds/.zarray', 'lon_bnds/.zattrs', 'lon_bnds/0.0',
    'time/.zarray', 'time/.zattrs', 'time/0', 'time/1', 'time/2',
    'time_bnds/.zarray', 'time_bnds/.zattrs', 'time_bnds/0.0', 'time_bnds/1.0', 'time_bnds/2.0'
}


class OptimizeDatasetTest(unittest.TestCase):
    def _clear_outputs(self):
        rimraf(INPUT_CUBE_PATH)
        rimraf(OUTPUT_CUBE_PATH)

    def setUp(self):
        self._clear_outputs()
        TEST_CUBE.to_zarr(INPUT_CUBE_PATH)
        self.assertEqual(INPUT_CUBE_FILE_SET, list_file_set(INPUT_CUBE_PATH))

    def tearDown(self):
        self._clear_outputs()

    def test_optimize_dataset(self):
        self._test_optimize_dataset(unchunk_coords=False, in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=False,
                                    expected_cons_time_bnds=False)

    def test_optimize_dataset_in_place(self):
        self._test_optimize_dataset(unchunk_coords=False, in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=False,
                                    expected_cons_time_bnds=False)

    def test_optimize_dataset_unchunk_coords_true(self):
        self._test_optimize_dataset(unchunk_coords=True, in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_true_in_place(self):
        self._test_optimize_dataset(unchunk_coords=True, in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_str(self):
        self._test_optimize_dataset(unchunk_coords='time', in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=False)

    def test_optimize_dataset_unchunk_coords_str_in_place(self):
        self._test_optimize_dataset(unchunk_coords='time', in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=False)

    def test_optimize_dataset_unchunk_coords_str_list(self):
        self._test_optimize_dataset(unchunk_coords=['time_bnds'], in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=False,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_str_list_in_place(self):
        self._test_optimize_dataset(unchunk_coords=['time_bnds'], in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=False,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_str_tuple(self):
        self._test_optimize_dataset(unchunk_coords=('time_bnds', 'time'), in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_str_tuple_in_place(self):
        self._test_optimize_dataset(unchunk_coords=('time_bnds', 'time'), in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=True)

    def _test_optimize_dataset(self, unchunk_coords: Union[bool, str, Sequence[str]], in_place: bool,
                               expected_output_path: str, expected_cons_time: bool = False,
                               expected_cons_time_bnds: bool = False):
        if not in_place:
            optimize_dataset(INPUT_CUBE_PATH, output_path=OUTPUT_CUBE_PATTERN,
                             in_place=in_place, unchunk_coords=unchunk_coords)
        else:
            optimize_dataset(INPUT_CUBE_PATH,
                             in_place=in_place, unchunk_coords=unchunk_coords)
        self._assert_consolidated(expected_output_path, expected_cons_time, expected_cons_time_bnds)

    def _assert_consolidated(self,
                             cube_path: str,
                             cons_time: bool = False,
                             cons_time_bnds: bool = False):
        self.assertTrue(os.path.isdir(cube_path))
        expected_files = set(INPUT_CUBE_FILE_SET)
        expected_files.add('.zmetadata')
        if cons_time:
            expected_files.remove('time/1')
            expected_files.remove('time/2')
        if cons_time_bnds:
            expected_files.remove('time_bnds/1.0')
            expected_files.remove('time_bnds/2.0')
        self.assertEqual(expected_files, list_file_set(cube_path))

    def test_failures(self):
        with self.assertRaises(RuntimeError) as cm:
            optimize_dataset('pippo', in_place=True, exception_type=RuntimeError)
        self.assertEqual('Input path must point to ZARR dataset directory.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            optimize_dataset(INPUT_CUBE_PATH, exception_type=RuntimeError)
        self.assertEqual('Output path must be given.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            optimize_dataset(INPUT_CUBE_PATH, output_path=INPUT_CUBE_PATH, exception_type=RuntimeError)
        self.assertEqual('Output path already exists.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            optimize_dataset(INPUT_CUBE_PATH, output_path='./' + INPUT_CUBE_PATH, exception_type=RuntimeError)
        self.assertEqual('Output path already exists.', f'{cm.exception}')


def list_file_set(dir_path: str) -> Set[str]:
    """ Get set of all files in a directory and all of its sub-directories. """
    actual_files = set()
    for root, dirs, files in os.walk(dir_path):
        for f in files:
            rel_root = root[len(dir_path) + 1:]
            if rel_root:
                actual_files.add(rel_root + '/' + f)
            else:
                actual_files.add(f)
    return actual_files


# test/core/test_reproject.py

import unittest

import numpy as np
# noinspection PyUnresolvedReferences
from matplotlib import pyplot as plt
from numpy.testing import assert_array_almost_equal

from test.sampledata import create_highroc_dataset
from xcube.core.reproject import reproject_xy_to_wgs84

nan = np.nan


class ReprojectTest(unittest.TestCase):

    def test_reproject_xy_to_wgs84_highroc(self):
        dst_width = 12
        dst_height = 9

        dataset = create_highroc_dataset()
        proj_dataset = reproject_xy_to_wgs84(dataset,
                                             src_xy_var_names=('lon', 'lat'),
                                             src_xy_tp_var_names=('TP_longitude', 'TP_latitude'),
                                             src_xy_gcp_step=1,
                                             src_xy_tp_gcp_step=1,
                                             dst_size=(dst_width, dst_height))

        self.assertIsNotNone(proj_dataset)
        self.assertEqual(dict(lon=dst_width, lat=dst_height, bnds=2),
                         proj_dataset.sizes)

        self.assertIn('lon', proj_dataset)
        self.assertEqual(proj_dataset.lon.shape, (dst_width,))
        self.assertIn('lat', proj_dataset)
        self.assertEqual(proj_dataset.lat.shape, (dst_height,))

        self.assertIn('lon_bnds', proj_dataset)
        self.assertEqual(proj_dataset.lon_bnds.shape, (dst_width, 2))
        self.assertIn('lat_bnds', proj_dataset)
        self.assertEqual(proj_dataset.lat_bnds.shape, (dst_height, 2))

        expected_conc_chl = np.array([[7., 7., 11., 11., 11., 11., nan, nan, nan, nan, 5., 5.],
                                      [7., 7., 11., 11., 11., 11., nan, nan, nan, 21., 21., 21.],
                                      [5., 5., 10., 10., 10., 10., 2., 2., 2., 21., 21., 21.],
                                      [5., 5., 10., 10., 10., 2., 2., 2., 2., 21., 17., 17.],
                                      [5., 5., 10., 10., 10., 20., 20., 20., 20., 17., 17., 17.],
                                      [5., 16., 6., 6., 6., 20., 20., 20., 17., 17., nan, nan],
                                      [16., 16., 6., 6., 6., 20., nan, nan, nan, nan, nan, nan],
                                      [16., 16., 6., nan, nan, nan, nan, nan, nan, nan, nan, nan],
                                      [nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]],
                                     dtype=np.float64)
        self.assertIn('conc_chl', proj_dataset)
        # print(proj_dataset.conc_chl)
        self.assertEqual(proj_dataset.conc_chl.shape, (dst_height, dst_width))
        self.assertEqual(proj_dataset.conc_chl.dtype, np.float64)
        assert_array_almost_equal(proj_dataset.conc_chl, expected_conc_chl)

        expected_c2rcc_flags = np.array([[1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],
                                         [1., 1., 1., 1., 1., 1., 1., 1., 1., 2., 2., 2.],
                                         [1., 1., 4., 4., 4., 4., 1., 1., 1., 2., 2., 2.],
                                         [1., 1., 4., 4., 4., 1., 1., 1., 1., 2., 1., 1.],
                                         [1., 1., 4., 4., 4., 1., 1., 1., 1., 1., 1., 1.],
                                         [1., 8., 1., 1., 1., 1., 1., 1., 1., 1., nan, nan],
                                         [8., 8., 1., 1., 1., 1., nan, nan, nan, nan, nan, nan],
                                         [8., 8., 1., nan, nan, nan, nan, nan, nan, nan, nan, nan],
                                         [nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]],
                                        dtype=np.float64)
        self.assertIn('c2rcc_flags', proj_dataset)
        # print(proj_dataset.c2rcc_flags)
        self.assertEqual(proj_dataset.c2rcc_flags.shape, (dst_height, dst_width))
        self.assertEqual(proj_dataset.c2rcc_flags.dtype, np.float64)
        assert_array_almost_equal(proj_dataset.c2rcc_flags, expected_c2rcc_flags)


# test/core/test_schema.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.new import new_cube
from xcube.core.schema import CubeSchema, get_dataset_chunks


class CubeSchemaTest(unittest.TestCase):

    def test_new(self):
        cube = new_cube(variables=dict(a=2, b=3, c=4))
        schema = CubeSchema.new(cube)
        self._assert_schema(schema, expected_shape=cube.a.shape)

    def test_new_chunked(self):
        cube = new_cube(variables=dict(a=2, b=3, c=4))
        cube = cube.chunk(dict(time=1, lat=90, lon=90))
        schema = CubeSchema.new(cube)
        self._assert_schema(schema, expected_shape=cube.a.shape, expected_chunks=(1, 90, 90))

    def test_xarray_accessor(self):
        # noinspection PyUnresolvedReferences
        import xcube.core.xarray
        cube = new_cube(variables=dict(a=2, b=3, c=4))
        schema = cube.xcube.schema
        self._assert_schema(schema, expected_shape=cube.a.shape)

    def _assert_schema(self, schema: CubeSchema, expected_shape=None, expected_chunks=None):
        self.assertEqual(3, schema.ndim)
        self.assertEqual(expected_shape, schema.shape)
        self.assertEqual(expected_chunks, schema.chunks)
        self.assertIn('time', schema.coords)
        self.assertIn('lat', schema.coords)
        self.assertIn('lon', schema.coords)
        self.assertEqual(('time', 'lat', 'lon'), schema.dims)
        self.assertEqual('time', schema.time_dim)
        self.assertEqual('lon', schema.x_name)
        self.assertEqual('lat', schema.y_name)
        self.assertEqual('time', schema.time_name)
        self.assertEqual('lon', schema.x_dim)
        self.assertEqual('lat', schema.y_dim)
        self.assertEqual('time', schema.time_dim)
        self.assertEqual(expected_shape[-1], schema.x_size)
        self.assertEqual(expected_shape[-2], schema.y_size)
        self.assertEqual(expected_shape[0], schema.time_size)
        self.assertEqual(expected_shape[-1], schema.x_var.size)
        self.assertEqual(expected_shape[-2], schema.y_var.size)
        self.assertEqual(expected_shape[0], schema.time_var.size)

    def test_repr_html(self):
        cube = new_cube(variables=dict(a=2, b=3, c=4))
        cube = cube.chunk(dict(time=1, lat=90, lon=90))
        schema = CubeSchema.new(cube)
        self.assertEqual("<table>"
                         "<tr><td>Shape:</td><td>(5, 180, 360)</td></tr>"
                         "<tr><td>Chunk sizes:</td><td>(1, 90, 90)</td></tr>"
                         "<tr><td>Dimensions:</td><td>('time', 'lat', 'lon')</td></tr>"
                         "</table>",
                         schema._repr_html_())

    def test_constructor_with_invalid_args(self):
        cube = new_cube(variables=dict(t=273))
        schema = CubeSchema.new(cube)
        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(None, schema.coords)
        self.assertEqual('shape must be a sequence of integer sizes',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(schema.shape, None)
        self.assertEqual('coords must be a mapping from dimension names to label arrays',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(schema.shape, cube.coords, x_name=None)
        self.assertEqual('x_name must be given',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(schema.shape, cube.coords, y_name=None)
        self.assertEqual('y_name must be given',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(schema.shape, cube.coords, time_name=None)
        self.assertEqual('time_name must be given',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape[1:], schema.coords)
        self.assertEqual('shape must have at least three dimensions',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape, schema.coords, dims=('lat', 'lon'))
        self.assertEqual('dims must have same length as shape',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape, schema.coords, dims=('lat', 'lon', 'time'))
        self.assertEqual("the first dimension in dims must be 'time'",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape, schema.coords, dims=('time', 'lon', 'lat'))
        self.assertEqual("the last two dimensions in dims must be 'lat' and 'lon'",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape, schema.coords, dims=schema.dims, chunks=(90, 90))
        self.assertEqual("chunks must have same length as shape",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            coords = dict(schema.coords)
            del coords['lat']
            CubeSchema(schema.shape, coords, dims=schema.dims, chunks=(1, 90, 90))
        self.assertEqual("missing variables 'lon', 'lat', 'time' in coords",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            coords = dict(schema.coords)
            lat = coords['lat']
            coords['lat'] = xr.DataArray(lat.values.reshape((1, len(lat))), dims=('b', lat.dims[0]), attrs=lat.attrs)
            CubeSchema(schema.shape, coords, dims=schema.dims, chunks=(1, 90, 90))
        self.assertEqual("variables 'lon', 'lat', 'time' in coords must be 1-D",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            coords = dict(schema.coords)
            lat = coords['lat']
            coords['lat'] = xr.DataArray(lat.values[1:], dims=('lat',), attrs=lat.attrs)
            CubeSchema(schema.shape, coords, dims=schema.dims, chunks=(1, 90, 90))
        self.assertEqual("number of labels of 'lat' in coords does not match shape",
                         f'{cm.exception}')

    def test_new_with_cube(self):
        cube = new_cube()
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("cube is empty",
                         f'{cm.exception}')

        cube = new_cube()
        del cube.coords['lon']
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("cube has no valid spatial coordinate variables",
                         f'{cm.exception}')

        cube = new_cube()
        del cube.coords['time']
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("cube has no valid time coordinate variable",
                         f'{cm.exception}')

        cube = new_cube(variables=dict(a=1, b=2))
        cube['c'] = xr.DataArray(np.array([1, 2, 3, 4, 5]), dims=('q',))
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("all variables must have same dimensions, but variable 'c' has dimensions ('q',)",
                         f'{cm.exception}')

        cube = new_cube(variables=dict(a=1, b=2))
        cube = cube.chunk(dict(time=1, lat=90, lon=90))
        cube['b'] = cube['b'].chunk(dict(time=1, lat=45, lon=90))
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("all variables must have same chunks, but variable 'b' has chunks (1, 45, 90)",
                         f'{cm.exception}')

        cube = new_cube(variables=dict(a=1, b=2))
        cube = cube.chunk(dict(time=1, lat=(44, 43, 46, 47), lon=90))
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("dimension 'lat' of variable 'a' has chunks of different sizes: (44, 43, 46, 47)",
                         f'{cm.exception}')


class GetDatasetChunksTest(unittest.TestCase):

    def test_empty_dataset(self):
        dataset = xr.Dataset()
        self.assertEqual({}, get_dataset_chunks(dataset))

    def test_only_coords_dataset(self):
        dataset = xr.Dataset(
            coords=dict(
                x=xr.DataArray(np.linspace(0, 1, 100), dims='x').chunk(10),
                y=xr.DataArray(np.linspace(0, 1, 100), dims='y').chunk(20),
            )
        )
        self.assertEqual({}, get_dataset_chunks(dataset))

    def test_different_chunks(self):
        dataset = xr.Dataset(data_vars=dict(
            a=xr.DataArray(
                np.linspace(0, 1, 100 * 100).reshape((100, 100)),
                dims=('y', 'x')
            ).chunk((30, 10)),
            b=xr.DataArray(
                np.linspace(0, 1, 100 * 100).reshape((100, 100)),
                dims=('y', 'x')
            ).chunk((25, 15)),
            c=xr.DataArray(
                np.linspace(0, 1, 100 * 100).reshape((1, 100, 100)),
                dims=('time', 'y', 'x')
            ).chunk((1, 25, 10)),
            d=xr.DataArray(  # d is not chunked!
                np.linspace(0, 1, 100 * 100).reshape((1, 100, 100)),
                dims=('time', 'y', 'x')
            ),
        ))
        self.assertEqual(
            {'time': 1, 'x': 10, 'y': 25},
            get_dataset_chunks(dataset)
        )


# test/core/test_select.py

import itertools
import json
import math
import unittest
from collections.abc import MutableMapping
from typing import Dict, KeysView, Iterator, Sequence, Any

import cftime
import numpy as np
import xarray as xr

from test.sampledata import create_highroc_dataset
from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.select import select_spatial_subset
from xcube.core.select import select_subset
from xcube.core.select import select_temporal_subset
from xcube.core.select import select_variables_subset


class SelectVariablesSubsetTest(unittest.TestCase):
    def test_select_variables_subset_all(self):
        ds1 = create_highroc_dataset()
        # noinspection PyTypeChecker
        ds2 = select_variables_subset(ds1, None)
        self.assertIs(ds2, ds1)
        ds2 = select_variables_subset(ds1, ds1.data_vars.keys())
        self.assertIs(ds2, ds1)

    def test_select_variables_subset_none(self):
        ds1 = create_highroc_dataset()
        ds2 = select_variables_subset(ds1, [])
        self.assertEqual(0, len(ds2.data_vars))
        ds2 = select_variables_subset(ds1, ['bibo'])
        self.assertEqual(0, len(ds2.data_vars))

    def test_select_variables_subset_some(self):
        ds1 = create_highroc_dataset()
        self.assertEqual(36, len(ds1.data_vars))
        ds2 = select_variables_subset(ds1, ['conc_chl', 'c2rcc_flags', 'rtoa_10'])
        self.assertEqual(3, len(ds2.data_vars))


class SelectSpatialSubsetTest(unittest.TestCase):
    def test_select_spatial_subset_all_ij_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, ij_bbox=(0, 0, 4, 3))
        self.assertIs(ds2, ds1)

    def test_select_spatial_subset_some_ij_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, ij_bbox=(1, 1, 4, 3))
        self.assertEqual((2, 3), ds2.conc_chl.shape)

    def test_select_spatial_subset_none_ij_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, ij_bbox=(5, 6, 7, 8))
        self.assertEqual(None, ds2)
        ds2 = select_spatial_subset(ds1, ij_bbox=(-6, -4, 2, 2))
        self.assertEqual(None, ds2)

    def test_select_spatial_subset_all_xy_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, xy_bbox=(7.9, 53.9, 12., 56.4))
        self.assertIs(ds2, ds1)

    def test_select_spatial_subset_some_xy_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, xy_bbox=(8., 55, 10., 56.))
        self.assertEqual((3, 3), ds2.conc_chl.shape)

    def test_select_spatial_subset_none_xy_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, xy_bbox=(13., 57., 15., 60.))
        self.assertEqual(None, ds2)
        ds2 = select_spatial_subset(ds1, xy_bbox=(5.5, 55, 6.5, 56))
        self.assertEqual(None, ds2)

    def test_select_spatial_subset_invalid_params(self):
        ds1 = create_highroc_dataset()
        with self.assertRaises(ValueError) as cm:
            select_spatial_subset(ds1, ij_bbox=(5, 6, 7, 8), xy_bbox=(0., 0., 1., 2.))
        self.assertEqual("Only one of ij_bbox and xy_bbox can be given", f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            select_spatial_subset(ds1)
        self.assertEqual("One of ij_bbox and xy_bbox must be given", f'{cm.exception}')

    def test_select_spatial_subset_descending_y_param(self):
        ds1 = new_cube(inverse_y=True)
        ds2 = select_spatial_subset(ds1, xy_bbox=(40., 40., 42., 42.))
        self.assertEqual(((2,), (2,)), (ds2.lon.shape, ds2.lat.shape))

    def test_select_spatial_subset_with_gm(self):
        ds1 = new_cube(inverse_y=True)
        ds2 = select_spatial_subset(ds1, xy_bbox=(40., 40., 42., 42.),
                                    grid_mapping=GridMapping.from_dataset(ds1))
        self.assertEqual(((2,), (2,)), (ds2.lon.shape, ds2.lat.shape))
        ds2 = select_spatial_subset(ds1, xy_bbox=(40., 40., 42., 42.),
                                    geo_coding=GridMapping.from_dataset(ds1))
        self.assertEqual(((2,), (2,)), (ds2.lon.shape, ds2.lat.shape))


class SelectTemporalSubsetTest(unittest.TestCase):
    def test_invalid_dataset(self):
        ds1 = xr.Dataset(dict(CHL=xr.DataArray([[1, 2], [2, 3]], dims=('lat', 'lon'))))
        with self.assertRaises(ValueError) as cm:
            select_temporal_subset(ds1, time_range=('2010-01-02', '2010-01-04'))
        self.assertEqual('cannot compute temporal subset: variable "time" not found in dataset',
                         f'{cm.exception}')

    def test_no_subset_for_open_interval(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8))
        ds2 = select_temporal_subset(ds1, time_range=(None, None))
        self.assertIs(ds2, ds1)

    def test_subset_for_closed_interval(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8))
        ds2 = select_temporal_subset(ds1, time_range=('2010-01-02', '2010-01-04'))
        self.assertIsNot(ds2, ds1)
        np.testing.assert_equal(np.array(['2010-01-02T12:00:00.000000000',
                                          '2010-01-03T12:00:00.000000000',
                                          '2010-01-04T12:00:00.000000000'],
                                         dtype='datetime64[ns]'),
                                ds2.time.values)

    def test_subset_for_upward_open_interval(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8))
        ds2 = select_temporal_subset(ds1, time_range=('2010-01-03', None))
        self.assertIsNot(ds2, ds1)
        np.testing.assert_equal(np.array(['2010-01-03T12:00:00.000000000',
                                          '2010-01-04T12:00:00.000000000',
                                          '2010-01-05T12:00:00.000000000'],
                                         dtype='datetime64[ns]'),
                                ds2.time.values)

    def test_subset_for_downward_open_interval(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8))
        ds2 = select_temporal_subset(ds1, time_range=(None, '2010-01-03'))
        self.assertIsNot(ds2, ds1)
        np.testing.assert_equal(np.array(['2010-01-01T12:00:00.000000000',
                                          '2010-01-02T12:00:00.000000000',
                                          '2010-01-03T12:00:00.000000000'],
                                         dtype='datetime64[ns]'),
                                ds2.time.values)

    def test_cf_time_subset(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8),
                       use_cftime=True,
                       time_dtype=None,
                       time_units='days since 1950-01-01',
                       time_calendar='julian')
        ds2 = select_temporal_subset(ds1, time_range=('2010-01-02', '2010-01-04'))
        self.assertIsNot(ds2, ds1)
        np.testing.assert_equal(np.array([cftime.DatetimeJulian(2010, 1, 2, 12, 0, 0),
                                          cftime.DatetimeJulian(2010, 1, 3, 12, 0, 0),
                                          cftime.DatetimeJulian(2010, 1, 4, 12, 0, 0)],
                                         dtype='object'),
                                ds2.time.values)


class SelectSubsetTest(unittest.TestCase):
    def test_all_params(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8), drop_bounds=True)
        ds2 = select_subset(ds1,
                            var_names=['analysed_sst'],
                            bbox=(10, 50, 15, 55),
                            time_range=('2010-01-02', '2010-01-04'))

        self.assertEqual({'analysed_sst'}, set(ds2.data_vars))
        self.assertEqual({'time', 'lat', 'lon'}, set(ds2.coords))

        np.testing.assert_almost_equal(np.array([10.5, 11.5, 12.5, 13.5, 14.5]),
                                       ds2.lon.values)

        np.testing.assert_almost_equal(np.array([50.5, 51.5, 52.5, 53.5, 54.5]),
                                       ds2.lat.values)

        np.testing.assert_equal(np.array(['2010-01-02T12:00:00.000000000',
                                          '2010-01-03T12:00:00.000000000',
                                          '2010-01-04T12:00:00.000000000'],
                                         dtype='datetime64[ns]'),
                                ds2.time.values)

    def test_xy_bbox_with_large_dataset(self):
        ds = self._new_large_dataset()
        ds_subset = select_spatial_subset(ds, xy_bbox=(0., 0., 5.0, 2.5))
        self._assert_large_dataset_subset(ds_subset)

    def test_ij_bbox_with_large_dataset(self):
        ds = self._new_large_dataset()
        ds_subset = select_spatial_subset(ds, ij_bbox=(129600 // 2, 64800 // 2, 129600 // 2 + 1800, 64800 // 2 + 900))
        self._assert_large_dataset_subset(ds_subset)

    def _new_large_dataset(self):
        ds = new_virtual_dataset(lon_size=129600, lat_size=64800, time_size=5,
                                 lon_chunk=1296, lat_chunk=648, time_chunk=1)
        # print(ds)
        self.assertIsInstance(ds, xr.Dataset)
        self.assertIn('conc_chl', ds)
        self.assertEqual(('time', 'lat', 'lon'), ds.conc_chl.dims)
        self.assertEqual((5, 64800, 129600), ds.conc_chl.shape)
        self.assertEqual((5 * (1,), 100 * (648,), 100 * (1296,)), ds.conc_chl.chunks)
        return ds

    def _assert_large_dataset_subset(self, ds_subset):
        # print(ds_subset)
        self.assertIsInstance(ds_subset, xr.Dataset)
        self.assertIn('conc_chl', ds_subset)
        self.assertEqual(('time', 'lat', 'lon'), ds_subset.conc_chl.dims)
        self.assertEqual((5, 900, 1800), ds_subset.conc_chl.shape)
        self.assertEqual(((1, 1, 1, 1, 1), (648, 252), (1296, 504)), ds_subset.conc_chl.chunks)


def new_virtual_dataset(lon_size: int = 360,
                        lat_size: int = 180,
                        time_size: int = 5,
                        lon_chunk: int = None,
                        lat_chunk: int = None,
                        time_chunk: int = None):
    lon_res = 360. / lon_size
    lat_res = 180. / lat_size

    lon_values = np.linspace(-180. + lon_res / 2, 180. - lon_res / 2, lon_size)
    lat_values = np.linspace(-90. + lat_res / 2, 90. - lat_res / 2, lat_size)
    day1 = 365 * (2021 - 1970)
    time_values = np.linspace(day1, day1 + time_size, time_size, dtype='int64')

    chunk_store = VirtualChunkStore()
    chunk_store.add_unchunked_array('lon', ['lon'], lon_values)
    chunk_store.add_unchunked_array('lat', ['lat'], lat_values)
    chunk_store.add_unchunked_array('time', ['time'], time_values, attrs={
        "calendar": "gregorian",
        "units": "days since 1970-01-01"
    })
    chunk_store.add_chunked_array('conc_chl',
                                  ['time', 'lat', 'lon'],
                                  [time_size, lat_size, lon_size],
                                  [time_chunk or time_size, lat_chunk or lat_size, lon_chunk or lon_size])

    return xr.open_zarr(chunk_store)


class VirtualChunkStore(MutableMapping):

    def __init__(self, entries: Dict[str, bytes] = None):
        if entries:
            self._entries: Dict[str, bytes] = dict(entries)
        else:
            self._entries: Dict[str, bytes] = {
                ".zgroup": bytes(json.dumps({"zarr_format": 2}, indent=2), encoding='utf-8'),
                ".zattrs": bytes(json.dumps({}, indent=2), encoding='utf-8'),
            }

    def add_unchunked_array(self,
                            name: str,
                            dims: Sequence[str],
                            values: np.ndarray,
                            attrs: Dict[str, Any] = None):
        self._entries.update(self.get_array_entries_unchunked(name, dims, values, attrs=attrs))

    def add_chunked_array(self,
                          name: str,
                          dims: Sequence[str],
                          shape: Sequence[int],
                          chunks: Sequence[int],
                          value: float = None):
        self._entries.update(self.get_array_entries_chunked(name, dims, shape, chunks, value=value))

    def keys(self) -> KeysView[str]:
        return self._entries.keys()

    def __iter__(self) -> Iterator[str]:
        return iter(self._entries.keys())

    def __len__(self) -> int:
        return len(self._entries)

    def __contains__(self, key) -> bool:
        return key in self._entries

    def __getitem__(self, key: str) -> bytes:
        return self._entries[key]

    def __setitem__(self, key: str, value: bytes) -> None:
        self._entries[key] = value

    def __delitem__(self, key: str) -> None:
        del self._entries[key]

    @classmethod
    def get_array_entries_unchunked(cls,
                                    name: str,
                                    dims: Sequence[str],
                                    values: np.ndarray,
                                    attrs: Dict[str, Any] = None) -> Dict[str, bytes]:

        zarray = {
            "zarr_format": 2,
            "chunks": list(values.shape),
            "shape": list(values.shape),
            "dtype": str(values.dtype.str),
            "order": "C",
            "fill_value": None,
            "compressor": None,
            "filters": None,
        }

        zattrs = {
            # For xarray
            "_ARRAY_DIMENSIONS": dims,
        }
        if attrs:
            zattrs.update(attrs)

        return {
            name + '/.zarray': bytes(json.dumps(zarray, indent=2), encoding='utf-8'),
            name + '/.zattrs': bytes(json.dumps(zattrs, indent=2), encoding='utf-8'),
            name + '/0': values.tobytes(order="C")
        }

    @classmethod
    def get_array_entries_chunked(cls,
                                  name: str,
                                  dims: Sequence[str],
                                  shape: Sequence[int],
                                  chunks: Sequence[int],
                                  value: float = None) -> Dict[str, bytes]:
        dtype = np.dtype('float64')

        zarray = {
            "zarr_format": 2,
            "dtype": str(dtype.str),
            "shape": list(shape),
            "chunks": list(chunks),
            "order": "C",
            "fill_value": "NaN",
            "compressor": None,
            "filters": None,
        }

        zattrs = {
            # For xarray
            "_ARRAY_DIMENSIONS": dims
        }

        entries = {
            name + '/.zarray': bytes(json.dumps(zarray, indent=2), encoding='utf-8'),
            name + '/.zattrs': bytes(json.dumps(zattrs, indent=2), encoding='utf-8'),
        }

        if value is not None:
            values = np.full(shape, value, dtype=dtype)
            data = values.tobytes(order="C")
            chunk_indices = [tuple(range(math.ceil(s / c))) for s, c in zip(shape, chunks)]
            for index in itertools.product(*chunk_indices):
                key = f'{name}/{"/".join(index)}'
                entries[key] = data

        return entries


# test/core/test_subsampling.py

#  The MIT License (MIT)
#  Copyright (c) 2022 by the xcube development team and contributors
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.

import unittest

import numpy as np
import xarray as xr

from xcube.core.new import new_cube
from xcube.core.subsampling import find_agg_method
from xcube.core.subsampling import get_dataset_subsampling_slices
from xcube.core.subsampling import subsample_dataset


class SubsampleDatasetTest(unittest.TestCase):

    def setUp(self) -> None:
        test_data_1 = np.array([[1, 2, 3, 4, 5, 6],
                                [2, 3, 4, 5, 6, 7],
                                [3, 4, 5, 6, 7, 8],
                                [4, 5, 6, 7, 8, 9],
                                [1, 2, 3, 4, 5, 6]], dtype=np.int16)
        test_data_1 = np.stack([test_data_1, test_data_1 + 10])
        test_data_2 = 0.1 * test_data_1
        self.dataset = new_cube(width=6,  # even
                                height=5,  # odd
                                x_name='x',
                                y_name='y',
                                x_start=0.5,
                                y_start=1.5,
                                x_res=1.,
                                y_res=1.,
                                time_periods=2,
                                crs='CRS84',
                                crs_name='spatial_ref',
                                variables=dict(var_1=test_data_1,
                                               var_2=test_data_2))

    def test_get_dataset_subsampling_slices(self):
        slices = get_dataset_subsampling_slices(self.dataset,
                                                step=2)
        self.assertEqual(
            {
                'var_1': (slice(None, None, None),
                          slice(None, None, 2),
                          slice(None, None, 2)),
                'var_2': (slice(None, None, None),
                          slice(None, None, 2),
                          slice(None, None, 2))
            },
            slices
        )

    def test_subsample_dataset_none(self):
        subsampled_dataset = subsample_dataset(self.dataset,
                                               step=2,
                                               agg_methods=None)
        self.assert_subsampling_ok(
            subsampled_dataset,
            np.array([
                [[1, 3, 5],
                 [3, 5, 7],
                 [1, 3, 5]],

                [[11, 13, 15],
                 [13, 15, 17],
                 [11, 13, 15]]
            ], dtype=np.int16),
            np.array([
                [[0.2, 0.4, 0.6],
                 [0.4, 0.6, 0.8],
                 [0.15, 0.35, 0.55]],

                [[1.2, 1.4, 1.6],
                 [1.4, 1.6, 1.8],
                 [1.15, 1.35, 1.55]]
            ], dtype=np.float64),
            np.array([1., 3., 5.]),
            np.array([2., 4., 6.]),
        )

    def test_subsample_dataset_first(self):
        subsampled_dataset = subsample_dataset(self.dataset,
                                               step=2,
                                               agg_methods='first')
        self.assert_subsampling_ok(
            subsampled_dataset,
            np.array([
                [[1, 3, 5],
                 [3, 5, 7],
                 [1, 3, 5]],

                [[11, 13, 15],
                 [13, 15, 17],
                 [11, 13, 15]]
            ], dtype=np.int16),
            np.array([
                [[0.1, 0.3, 0.5],
                 [0.3, 0.5, 0.7],
                 [0.1, 0.3, 0.5]],

                [[1.1, 1.3, 1.5],
                 [1.3, 1.5, 1.7],
                 [1.1, 1.3, 1.5]]
            ], dtype=np.float64),
            np.array([1., 3., 5.]),
            np.array([2., 4., 6.]),
        )

    def test_subsample_dataset_mean(self):
        subsampled_dataset = subsample_dataset(self.dataset,
                                               step=2,
                                               agg_methods='mean')
        self.assert_subsampling_ok(
            subsampled_dataset,
            np.array([
                [[2, 4, 6],
                 [4, 6, 8],
                 [1, 3, 5]],

                [[12, 14, 16],
                 [14, 16, 18],
                 [11, 13, 15]]
            ], dtype=np.int16),
            np.array([
                [[0.2, 0.4, 0.6],
                 [0.4, 0.6, 0.8],
                 [0.15, 0.35, 0.55]],

                [[1.2, 1.4, 1.6],
                 [1.4, 1.6, 1.8],
                 [1.15, 1.35, 1.55]]
            ], dtype=np.float64),
            np.array([1., 3., 5.]),
            np.array([2., 4., 6.]),
        )

    def test_subsample_dataset_max(self):
        subsampled_dataset = subsample_dataset(self.dataset,
                                               step=2,
                                               agg_methods='max')
        self.assert_subsampling_ok(
            subsampled_dataset,
            np.array([
                [[3, 5, 7],
                 [5, 7, 9],
                 [2, 4, 6]],

                [[13, 15, 17],
                 [15, 17, 19],
                 [12, 14, 16]]
            ], dtype=np.int16),
            np.array([
                [[0.3, 0.5, 0.7],
                 [0.5, 0.7, 0.9],
                 [0.2, 0.4, 0.6]],

                [[1.3, 1.5, 1.7],
                 [1.5, 1.7, 1.9],
                 [1.2, 1.4, 1.6]]
            ], dtype=np.float64),
            np.array([1., 3., 5.]),
            np.array([2., 4., 6.]),
        )

    def assert_subsampling_ok(self,
                              subsampled_dataset: xr.Dataset,
                              expected_var_1: np.ndarray,
                              expected_var_2: np.ndarray,
                              expected_x: np.ndarray,
                              expected_y: np.ndarray):
        self.assertIsInstance(subsampled_dataset, xr.Dataset)
        self.assertIn('spatial_ref',
                      subsampled_dataset)
        self.assertIn('grid_mapping_name',
                      subsampled_dataset.spatial_ref.attrs)

        self.assertIn('x',
                      subsampled_dataset.coords)
        self.assertIn('y',
                      subsampled_dataset.coords)

        np.testing.assert_array_equal(
            subsampled_dataset.x.values,
            expected_x
        )
        np.testing.assert_array_almost_equal(
            subsampled_dataset.y.values,
            expected_y
        )

        self.assertIn('var_1',
                      subsampled_dataset.data_vars)
        self.assertIn('var_2',
                      subsampled_dataset.data_vars)

        self.assertIn('grid_mapping',
                      subsampled_dataset.var_1.attrs)
        self.assertIn('grid_mapping',
                      subsampled_dataset.var_2.attrs)

        self.assertEqual(expected_var_1.dtype,
                         subsampled_dataset.var_1.dtype)
        self.assertEqual(expected_var_2.dtype,
                         subsampled_dataset.var_2.dtype)

        np.testing.assert_array_equal(
            subsampled_dataset.var_1.values,
            expected_var_1
        )
        np.testing.assert_array_almost_equal(
            subsampled_dataset.var_2.values,
            expected_var_2
        )

    # noinspection PyTypeChecker
    def test_find_agg_method(self):
        for m in ('first', 'min', 'max', 'mean', 'median'):
            self.assertEqual(m, find_agg_method(m, 'var_1', np.uint8))
            self.assertEqual(m, find_agg_method(m, 'var_2', np.float32))

        for m in ({'*': 'max'}, {'var_*': 'max'}):
            self.assertEqual('max',
                             find_agg_method(m, 'var_1', np.uint8))
            self.assertEqual('max',
                             find_agg_method(m, 'var_2', np.float32))

        for m in (None, 'auto'):
            self.assertEqual('first', find_agg_method(m, 'var_1', np.uint8))
            self.assertEqual('mean', find_agg_method(m, 'var_2', np.float32))

        for m in ({'*': None}, {'*': 'auto'},
                  {'var_*': None}, {'var_*': 'auto'}):
            self.assertEqual('first', find_agg_method(m, 'var_1', np.uint8))
            self.assertEqual('mean', find_agg_method(m, 'var_2', np.float32))


# test/core/test_tile.py

import unittest

import numpy as np
import pandas as pd
import xarray as xr

from xcube.core.tile import get_var_valid_range
from xcube.core.tile import parse_non_spatial_labels
from xcube.core.tile import _ensure_time_compatible


class GetVarValidRangeTest(unittest.TestCase):
    def test_from_valid_range(self):
        a = xr.DataArray(0, attrs=dict(valid_range=[-1, 1]))
        self.assertEqual((-1, 1),
                         get_var_valid_range(a))

    def test_from_valid_min_max(self):
        a = xr.DataArray(0, attrs=dict(valid_min=-1, valid_max=1))
        self.assertEqual((-1, 1),
                         get_var_valid_range(a))

    def test_from_valid_min(self):
        a = xr.DataArray(0, attrs=dict(valid_min=-1))
        self.assertEqual((-1, np.inf),
                         get_var_valid_range(a))

    def test_from_valid_max(self):
        a = xr.DataArray(0, attrs=dict(valid_max=1))
        self.assertEqual((-np.inf, 1),
                         get_var_valid_range(a))

    def test_from_nothing(self):
        a = xr.DataArray(0)
        self.assertEqual(None,
                         get_var_valid_range(a))


class ParseNonSpatialLabelsTest(unittest.TestCase):
    time = xr.DataArray(np.array(['2021-06-07 10:20:45',
                                  '2021-06-08 13:28:16',
                                  '2021-06-09 11:10:21'],
                                 dtype='datetime64'),
                        dims='time')
    lat = xr.DataArray(np.array([60, 61, 62], dtype=np.float64),
                       dims='lat')
    lon = xr.DataArray(np.array([4, 5, 6], dtype=np.float64),
                       dims='lon')
    coords = dict(time=time, lat=lat, lon=lon)

    dims = ('time', 'lat', 'lon')

    def test_use_first_not_given(self):
        labels = parse_non_spatial_labels(dict(),
                                          dims=self.dims,
                                          coords=self.coords)
        self.assertEqual(
            dict(time=np.array('2021-06-07 10:20:45', dtype='datetime64')),
            labels
        )

    def test_use_last_if_current(self):
        labels = parse_non_spatial_labels(dict(time='current'),
                                          dims=self.dims,
                                          coords=self.coords)
        self.assertEqual(
            dict(time=np.array('2021-06-09 11:10:21', dtype='datetime64')),
            labels
        )

    def test_use_given_if_value_given(self):
        labels = parse_non_spatial_labels(dict(time='2021-06-08'),
                                          dims=self.dims,
                                          coords=self.coords)
        self.assertEqual(
            dict(time=np.array('2021-06-08', dtype='datetime64')),
            labels
        )

    def test_use_average_if_range_given(self):
        labels = parse_non_spatial_labels(dict(time='2021-06-06/2021-06-08'),
                                          dims=self.dims,
                                          coords=self.coords)
        self.assertEqual(
            dict(time=np.array('2021-06-07', dtype='datetime64')),
            labels
        )

    def test_use_slice_if_range_given(self):
        labels = parse_non_spatial_labels(dict(time='2021-06-06/2021-06-08'),
                                          dims=self.dims,
                                          coords=self.coords,
                                          allow_slices=True)
        self.assertEqual(
            dict(time=slice(np.array('2021-06-06', dtype='datetime64'),
                            np.array('2021-06-08', dtype='datetime64'))),
            labels
        )

    def test_invalid(self):
        with self.assertRaises(ValueError) as cm:
            parse_non_spatial_labels(dict(time='jetzt'),
                                     dims=self.dims,
                                     coords=self.coords)
        self.assertEqual("'jetzt' is not a valid value for dimension 'time'",
                         f'{cm.exception}')

    def test_ensure_timezone_naive(self):
        da_tznaive = xr.DataArray(
            np.zeros((3,3,3)),
            coords=self.coords,
            dims=self.dims)
        labels = parse_non_spatial_labels(dict(time='2000-01-02T00:00:00Z'),
                                          dims=da_tznaive.dims,
                                          coords=da_tznaive.coords,
                                          var=da_tznaive)
        self.assertIsNone(pd.Timestamp(labels['time']).tzinfo)


class EnsureTimeCompatibleTest(unittest.TestCase):
    da_tznaive = xr.DataArray(
        np.arange(1, 4),
        coords=dict(time=np.arange('2000-01-01', '2000-01-04',
                                   dtype=np.datetime64)),
        dims=['time'])
    da_tzaware = xr.DataArray(
        np.arange(1, 4),
        coords=dict(time=pd.date_range("2000-01-01", "2000-01-03", tz='UTC')),
        dims=['time'])
    labels_tznaive = dict(time='2000-01-02')
    labels_tzaware = dict(time='2000-01-02T00:00:00Z')

    def test_both_tznaive(self):
        self.assertEqual(self.labels_tznaive,
                         _ensure_time_compatible(self.da_tznaive,
                                                 self.labels_tznaive))

    def test_both_tzaware(self):
        self.assertEqual(self.labels_tzaware,
                         _ensure_time_compatible(self.da_tzaware,
                                                 self.labels_tzaware))

    def test_tznaive_array_tzaware_indexer(self):
        self.assertTrue(
            _are_times_equal(
                self.labels_tznaive,
                _ensure_time_compatible(self.da_tznaive,
                                        self.labels_tzaware)))


def _are_times_equal(labels1, labels2):
    return pd.Timestamp(labels1['time']) == pd.Timestamp(labels2['time'])


# test/core/test_tile2.py

import unittest

import numpy as np
import pyproj
import xarray as xr

from xcube.core.mldataset import BaseMultiLevelDataset
from xcube.core.tile2 import compute_color_mapped_rgba_tile
from xcube.util.tilegrid2 import GEOGRAPHIC_CRS_NAME
from xcube.util.tilegrid2 import WEB_MERCATOR_CRS_NAME


class Tile2Test(unittest.TestCase):

    def test_compute_color_mapped_rgba_tile(self):
        crs_name = WEB_MERCATOR_CRS_NAME

        crs = pyproj.CRS.from_string(crs_name)
        geo_crs = pyproj.CRS.from_string(GEOGRAPHIC_CRS_NAME)
        transformer = pyproj.Transformer.from_crs(
            geo_crs, crs, always_xy=True
        )
        (x1, x2), (y1, y2) = transformer.transform(
            (-180, 180),
            (-85.051129, 85.051129)
        )

        a_data = np.array(
            [[
                [5, 1, 1, 1, 1, 2, 2, 2, 2, 6],
                [1, 5, 1, 1, 1, 2, 2, 2, 6, 2],
                [1, 1, 5, 1, 1, 2, 2, 6, 2, 2],
                [1, 1, 1, 5, 1, 2, 6, 2, 2, 2],
                [1, 1, 1, 1, 5, 6, 2, 2, 2, 2],
                [3, 3, 3, 3, 6, 5, 4, 4, 4, 4],
                [3, 3, 3, 6, 3, 4, 5, 4, 4, 4],
                [3, 3, 6, 3, 3, 4, 4, 5, 4, 4],
                [3, 6, 3, 3, 3, 4, 4, 4, 5, 4],
                [6, 3, 3, 3, 3, 4, 4, 4, 4, 5],
            ]],
            dtype=np.float32
        )
        a_data = np.where(a_data != 6, a_data, np.nan)
        h, w = a_data.shape[-2:]
        print(x1, x2)
        print(y1, y2)
        res = (x2 - x1) / w
        x1 += 0.5 * res
        x2 -= 0.5 * res
        y1 += 0.5 * res
        y2 -= 0.5 * res

        var_a = xr.DataArray(a_data, dims=['time', 'y', 'x'])
        spatial_ref = xr.DataArray(
            0, attrs=pyproj.CRS.from_string(crs_name).to_cf()
        )

        ds = xr.Dataset(
            data_vars=dict(
                var_a=var_a,
                spatial_ref=spatial_ref
            ),
            coords=dict(
                time=xr.DataArray(np.array([0]), dims='time'),
                y=xr.DataArray(np.linspace(y1, y2, h), dims='y'),
                x=xr.DataArray(np.linspace(x1, x2, w), dims='x')
            )
        )
        # ds = ds.chunk(dict(x=4, y=3))
        ml_ds = BaseMultiLevelDataset(ds)

        args = [
            ml_ds,
            'var_a',
            0, 0, 0
        ]
        kwargs = dict(
            crs_name=crs_name,
            tile_size=10,
            cmap_name="gray",
            value_range=(0, 10),
            non_spatial_labels={'time': 0},
            tile_enlargement=0
        )
        tile = compute_color_mapped_rgba_tile(*args, **kwargs, format='numpy')
        self.assertIsInstance(tile, np.ndarray)
        self.assertEqual(np.uint8, tile.dtype)
        self.assertEqual((10, 10, 4), tile.shape)
        r = tile[:, :, 0]
        g = tile[:, :, 1]
        b = tile[:, :, 2]
        a = tile[:, :, 3]
        np.testing.assert_equal(
            r,
            np.array(
                [[0, 0, 76, 76, 76, 76, 102, 102, 102, 128],
                 [76, 76, 0, 76, 76, 76, 102, 102, 128, 102],
                 [76, 76, 76, 0, 76, 76, 102, 128, 102, 102],
                 [76, 76, 76, 76, 0, 0, 128, 102, 102, 102],
                 [25, 25, 25, 25, 128, 128, 0, 51, 51, 51],
                 [25, 25, 25, 25, 128, 128, 0, 51, 51, 51],
                 [25, 25, 25, 128, 25, 25, 51, 0, 51, 51],
                 [25, 25, 128, 25, 25, 25, 51, 51, 0, 51],
                 [128, 128, 25, 25, 25, 25, 51, 51, 51, 0],
                 [128, 128, 25, 25, 25, 25, 51, 51, 51, 0]],
                dtype=np.uint8
            )
        )
        np.testing.assert_equal(r, g)
        np.testing.assert_equal(r, b)
        np.testing.assert_equal(
            a,
            np.array(
                [[0, 0, 255, 255, 255, 255, 255, 255, 255, 255],
                 [255, 255, 0, 255, 255, 255, 255, 255, 255, 255],
                 [255, 255, 255, 0, 255, 255, 255, 255, 255, 255],
                 [255, 255, 255, 255, 0, 0, 255, 255, 255, 255],
                 [255, 255, 255, 255, 255, 255, 0, 255, 255, 255],
                 [255, 255, 255, 255, 255, 255, 0, 255, 255, 255],
                 [255, 255, 255, 255, 255, 255, 255, 0, 255, 255],
                 [255, 255, 255, 255, 255, 255, 255, 255, 0, 255],
                 [255, 255, 255, 255, 255, 255, 255, 255, 255, 0],
                 [255, 255, 255, 255, 255, 255, 255, 255, 255, 0]],
                dtype=np.uint8
            )
        )

        tile = compute_color_mapped_rgba_tile(*args, format='png')
        self.assertIsInstance(tile, bytes)


# test/core/test_timecoord.py

import unittest

import numpy as np
import pandas as pd

from test.sampledata import create_highroc_dataset
from xcube.core.new import new_cube
from xcube.core.timecoord import add_time_coords
from xcube.core.timecoord import from_time_in_days_since_1970
from xcube.core.timecoord import get_end_time_from_attrs
from xcube.core.timecoord import get_start_time_from_attrs
from xcube.core.timecoord import get_time_range_from_attrs
from xcube.core.timecoord import get_time_range_from_data
from xcube.core.timecoord import timestamp_to_iso_string
from xcube.core.timecoord import to_time_in_days_since_1970
# from xcube.core.timecoord import find_datetime_format
# from xcube.core.timecoord import get_timestamp_from_string
# from xcube.core.timecoord import get_timestamps_from_string


class AddTimeCoordsTest(unittest.TestCase):

    def test_add_time_coords_point(self):
        dataset = create_highroc_dataset()
        dataset_with_time = add_time_coords(dataset, (365 * 47 + 20, 365 * 47 + 20))
        self.assertIsNot(dataset_with_time, dataset)
        self.assertIn('time', dataset_with_time)
        self.assertEqual(dataset_with_time.time.shape, (1,))
        self.assertNotIn('time_bnds', dataset_with_time)

    def test_add_time_coords_range(self):
        dataset = create_highroc_dataset()
        dataset_with_time = add_time_coords(dataset, (365 * 47 + 20, 365 * 47 + 21))
        self.assertIsNot(dataset_with_time, dataset)
        self.assertIn('time', dataset_with_time)
        self.assertEqual(dataset_with_time.time.shape, (1,))
        self.assertIn('time_bnds', dataset_with_time)
        self.assertEqual(dataset_with_time.time_bnds.shape, (1, 2))

    def test_to_time_in_days_since_1970(self):
        self.assertEqual(17324.5,
                         to_time_in_days_since_1970('201706071200'))
        self.assertEqual(17325.5,
                         to_time_in_days_since_1970('201706081200'))
        self.assertEqual(17690.5,
                         to_time_in_days_since_1970('2018-06-08 12:00'))
        self.assertEqual(17690.5,
                         to_time_in_days_since_1970('2018-06-08T12:00'))
        self.assertEqual(18173.42625622898,
                         to_time_in_days_since_1970('04-OCT-2019 10:13:48.538184'))

    def test_from_time_in_days_since_1970(self):
        self.assertEqual('2017-06-07T12:00:00.000000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('201706071200'))))
        self.assertEqual('2017-06-08T12:00:00.000000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('201706081200'))))
        self.assertEqual('2018-06-08T12:00:00.000000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('2018-06-08 12:00'))))
        self.assertEqual('2018-06-08T12:00:00.000000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('2018-06-08T12:00'))))
        self.assertEqual('2019-10-04T10:13:48.538000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('04-OCT-2019 10:13:48.538184'))))


class GetTimeRangeTest(unittest.TestCase):

    def test_get_time_range_from_data(self):
        cube = new_cube(drop_bounds=True)
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_with_irregular_data(self):
        cube = new_cube(drop_bounds=True,
                        time_freq='M')
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_with_irregular_data_and_no_metadata(self):
        cube = new_cube(drop_bounds=True,
                        time_freq='M')
        cube.attrs.pop('time_coverage_start')
        cube.attrs.pop('time_coverage_end')
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_cftime(self):
        cube = new_cube(drop_bounds=True,
                        use_cftime=True,
                        time_dtype=None)
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_with_irregular_cftime_data(self):
        cube = new_cube(drop_bounds=True,
                        time_freq='M',
                        use_cftime=True,
                        time_dtype=None)
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_with_irregular_cftime_data_and_no_metadata(self):
        cube = new_cube(drop_bounds=True,
                        time_freq='M',
                        use_cftime=True,
                        time_dtype=None)
        cube.attrs.pop('time_coverage_start')
        cube.attrs.pop('time_coverage_end')
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_time_named_t(self):
        cube = new_cube(drop_bounds=True, time_name='t')
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_additional_t_variable(self):
        import xarray as xr
        start_time_data = pd.date_range(start='2010-01-03T12:00:00',
                                        periods=5,
                                        freq='5D').values.astype(dtype='datetime64[s]')
        start_time = xr.DataArray(start_time_data, dims='time')
        end_time_data = pd.date_range(start='2010-01-07T12:00:00',
                                      periods=5,
                                      freq='5D').values.astype(dtype='datetime64[s]')
        end_time = xr.DataArray(end_time_data, dims='time')
        cube = new_cube(drop_bounds=True,
                        time_start='2010-01-05T12:00:00',
                        time_freq='5D',
                        variables=dict(start_time=start_time, end_time=end_time))
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-03T12:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-27T12:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_start_and_end_time_arrays(self):
        cube = new_cube(drop_bounds=True,
                        use_cftime=True,
                        time_dtype=None)
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_bounds(self):
        cube = new_cube()
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_attrs(self):
        cube = new_cube()
        time_range = get_time_range_from_attrs(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_start_time_from_attrs(self):
        cube = new_cube()
        start_time = get_start_time_from_attrs(cube)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(start_time).isoformat())

    def test_get_end_time_from_attrs(self):
        cube = new_cube()
        end_time = get_end_time_from_attrs(cube)
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(end_time).isoformat())


class TimestampToIsoStringTest(unittest.TestCase):
    def test_it_with_default_res(self):
        self.assertEqual("2018-09-05T00:00:00Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05")))
        self.assertEqual("2018-09-05T10:35:42Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42")))
        self.assertEqual("2018-09-05T10:35:42Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164")))
        self.assertEqual("2019-10-04T10:13:49Z",
                         timestamp_to_iso_string(pd.to_datetime("04-OCT-2019 10:13:48.538184")))

    def test_it_with_h_res(self):
        self.assertEqual("2018-09-05T00:00:00Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05"),
                                                 freq="H"))
        self.assertEqual("2018-09-05T11:00:00Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42"),
                                                 freq="H"))
        self.assertEqual("2018-09-05T11:00:00Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164"),
                                                 freq="H"))
        self.assertEqual("2019-10-04T10:00:00Z",
                         timestamp_to_iso_string(pd.to_datetime("04-OCT-2019 10:13:48.538184"),
                                                 freq="H"))


# class TimeStampsTest(unittest.TestCase):
#
#     def test_find_datetime_format(self):
#         dt_format, start_index, end_index = find_datetime_format('ftze20140305131415dgs')
#         self.assertEqual('%Y%m%d%H%M%S', dt_format)
#         self.assertEqual(4, start_index)
#         self.assertEqual(18, end_index)
#
#         dt_format, start_index, end_index = find_datetime_format('ftze201403051314dgs')
#         self.assertEqual('%Y%m%d%H%M', dt_format)
#         self.assertEqual(4, start_index)
#         self.assertEqual(16, end_index)
#
#         dt_format, start_index, end_index = find_datetime_format('ft2ze20140307dgs')
#         self.assertEqual('%Y%m%d', dt_format)
#         self.assertEqual(5, start_index)
#         self.assertEqual(13, end_index)
#
#         dt_format, start_index, end_index = find_datetime_format('ft2ze201512dgs')
#         self.assertEqual('%Y%m', dt_format)
#         self.assertEqual(5, start_index)
#         self.assertEqual(11, end_index)
#
#         dt_format, start_index, end_index = find_datetime_format('ft2s6ze2016dgs')
#         self.assertEqual('%Y', dt_format)
#         self.assertEqual(7, start_index)
#         self.assertEqual(11, end_index)
#
#     def test_get_timestamp_from_string(self):
#         timestamp = get_timestamp_from_string('ftze20140305131415dg0023s')
#         self.assertEqual(pd.Timestamp('2014-03-05T13:14:15'), timestamp)
#
#         timestamp = get_timestamp_from_string('ftze201403051314dgs')
#         self.assertEqual(pd.Timestamp('2014-03-05T13:14:00'), timestamp)
#
#         timestamp = get_timestamp_from_string('ftze20140305dgs')
#         self.assertEqual(pd.Timestamp('2014-03-05T00:00:00'), timestamp)
#
#         timestamp = get_timestamp_from_string('ftze201403dgs')
#         self.assertEqual(pd.Timestamp('2014-03-01T00:00:00'), timestamp)
#
#         timestamp = get_timestamp_from_string('ftze2014dgs')
#         self.assertEqual(pd.Timestamp('2014-01-01T00:00:00'), timestamp)
#
#     def test_get_timestamps_from_string(self):
#         timestamp_1, timestamp_2 = get_timestamps_from_string(
#             '20020401-20020406-ESACCI-L3C_AEROSOL-AEX-GOMOS_ENVISAT-AERGOM_5days-fv2.19.nc')
#         self.assertEqual(pd.Timestamp('2002-04-01'), timestamp_1)
#         self.assertEqual(pd.Timestamp('2002-04-06'), timestamp_2)
#
#         timestamp_1, timestamp_2 = get_timestamps_from_string(
#             '20020401-ESACCI-L3C_AEROSOL-AEX-GOMOS_ENVISAT-AERGOM_5days-fv2.19.nc')
#         self.assertEqual(pd.Timestamp('2002-04-01'), timestamp_1)
#         self.assertIsNone(timestamp_2)


# test/core/test_timeseries.py

import unittest
from typing import Set

import numpy as np
import pyproj
import xarray as xr

from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.timeseries import get_time_series

POINT_GEOMETRY = dict(type='Point', coordinates=[20.0, 10.0])
POLYGON_GEOMETRY = dict(type='Polygon', coordinates=[[[20.0, 10.0],
                                                      [20.0, 20.0],
                                                      [10.0, 20.0],
                                                      [10.0, 10.0],
                                                      [20.0, 10.0]]])


class GetTimeSeriesTest(unittest.TestCase):

    def test_point(self):
        ts_ds = get_time_series(self.cube, geometry=POINT_GEOMETRY)
        self.assert_dataset_ok(ts_ds, 1, {'A', 'B'})

    def test_polygon(self):
        ts_ds = get_time_series(self.cube, geometry=POLYGON_GEOMETRY)
        self.assert_dataset_ok(ts_ds, 100, {'A_mean', 'B_mean'})

    def test_polygon_with_grid_mapping(self):
        crs1 = pyproj.CRS.from_string('EPSG:4326')  # = Geographic
        crs2 = pyproj.CRS.from_string('+proj=mill +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +R_A +datum=WGS84 +units=m +no_defs')  # = World Miller
        t = pyproj.Transformer.from_crs(crs1, crs2, always_xy=True)
        cube = self.cube.rename(dict(lat='y', lon='x'))
        x, _ = t.transform(
            cube.x,
            np.zeros(cube.x.size)
        )
        _, y = t.transform(
            np.zeros(cube.y.size),
            cube.y,
        )
        cube = cube.assign_coords(
            x=xr.DataArray(x, dims='x'),
            y=xr.DataArray(y, dims='y'),
        )
        cube = cube.assign(crs=xr.DataArray(0, attrs=crs2.to_cf()))
        grid_mapping = GridMapping.from_dataset(cube)
        ts_ds = get_time_series(cube,
                                grid_mapping=grid_mapping,
                                geometry=POLYGON_GEOMETRY)
        self.assertIsInstance(ts_ds, xr.Dataset)
        # TODO (forman): we must actually get a valid
        #  result here, but we don't
        # self.assert_dataset_ok(ts_ds, 100, {'A_mean', 'B_mean'})

    def test_polygon_deprecated_cube_asserted(self):
        ts_ds = get_time_series(self.cube, geometry=POLYGON_GEOMETRY,
                                cube_asserted=True)
        self.assert_dataset_ok(ts_ds, 100, {'A_mean', 'B_mean'})

    def test_polygon_agg_median_mean_std(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                agg_methods=['mean', 'std', 'median'])
        self.assert_dataset_ok(
            ts_ds, 100,
            {'A_mean', 'A_median', 'A_std', 'B_mean', 'B_median', 'B_std'}
        )

    def test_polygon_agg_median_mean_std_groupby(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                agg_methods=['mean', 'std', 'median'],
                                use_groupby=True)
        self.assert_dataset_ok(
            ts_ds, 100,
            {'A_mean', 'A_median', 'A_std', 'B_mean', 'B_median', 'B_std'}
        )

    def test_polygon_agg_mean_count(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                agg_methods=['mean', 'count'])
        self.assert_dataset_ok(
            ts_ds, 100,
            {'A_mean', 'A_count', 'B_mean', 'B_count'}
        )

    def test_polygon_agg_mean_std_var_subs(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                var_names=['B'],
                                agg_methods=['mean', 'std'])
        self.assert_dataset_ok(ts_ds, 100, {'B_mean', 'B_std'})

    def test_no_vars(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                var_names=[])
        self.assertIsNone(ts_ds)

    def test_illegal_agg_methods(self):
        with self.assertRaises(ValueError) as cm:
            get_time_series(self.cube,
                            geometry=POLYGON_GEOMETRY,
                            agg_methods=['mean', 'median', 'stdev'])
        self.assertEqual('invalid aggregation method: stdev',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            get_time_series(self.cube,
                            geometry=POLYGON_GEOMETRY,
                            agg_methods=['median', 'stdev', 'avg'])
        self.assertEqual('invalid aggregation methods: avg, stdev',
                         f'{cm.exception}')

    def setUp(self):
        shape = 25, 180, 360
        dims = 'time', 'lat', 'lon'
        self.ts_a = np.linspace(1, 25, 25)
        self.ts_a_mean = np.linspace(1, 25, 25)
        self.ts_a_count = np.array(25 * [100])
        self.ts_a_std = np.array(25 * [0.0])
        self.ts_b = np.linspace(0, 1, 25)
        self.ts_b_mean = np.linspace(0, 1, 25)
        self.ts_b_count = np.array(25 * [100])
        self.ts_b_std = np.array(25 * [0.0])
        self.cube = new_cube(
            time_periods=25,
            variables=dict(
                A=xr.DataArray(
                    np.broadcast_to(self.ts_a.reshape((25, 1, 1)), shape),
                    dims=dims),
                B=xr.DataArray(
                    np.broadcast_to(self.ts_b.reshape((25, 1, 1)), shape),
                    dims=dims)
            )
        )
        self.cube = self.cube.chunk(chunks=dict(time=1, lat=180, lon=180))

    def assert_dataset_ok(self, ts_ds: xr.Dataset,
                          expected_max_number_of_observations: int = 0,
                          expected_var_names: Set = None):
        expected_var_names = expected_var_names or set()
        self.assertIsNotNone(ts_ds)
        self.assertEqual(expected_max_number_of_observations, ts_ds.attrs.get('max_number_of_observations'))
        self.assert_variable_ok(ts_ds, 'A', expected_var_names, self.ts_a)
        self.assert_variable_ok(ts_ds, 'A_mean', expected_var_names, self.ts_a_mean)
        self.assert_variable_ok(ts_ds, 'A_count', expected_var_names, self.ts_a_count)
        self.assert_variable_ok(ts_ds, 'A_std', expected_var_names, self.ts_a_std)
        self.assert_variable_ok(ts_ds, 'B', expected_var_names, self.ts_b)
        self.assert_variable_ok(ts_ds, 'B_mean', expected_var_names, self.ts_b_mean)
        self.assert_variable_ok(ts_ds, 'B_count', expected_var_names, self.ts_b_count)
        self.assert_variable_ok(ts_ds, 'B_std', expected_var_names, self.ts_b_std)

    def assert_variable_ok(self, ts_ds, name, expected_var_names: Set, expected_values):
        if name in expected_var_names:
            self.assertIn(name, ts_ds)
            self.assertEqual(('time',), ts_ds[name].dims)
            self.assertEqual(25, ts_ds[name].size)
            np.testing.assert_almost_equal(ts_ds[name].values, expected_values)
        else:
            self.assertNotIn(name, ts_ds)


# test/core/test_timeslice.py

import unittest

import numpy as np
import xarray as xr
import zarr

from test.core.diagnosticstore import DiagnosticStore, logging_observer
from xcube.core.chunk import chunk_dataset
from xcube.core.dsio import rimraf
from xcube.core.new import new_cube
from xcube.core.timeslice import find_time_slice, append_time_slice, insert_time_slice, replace_time_slice


class TimeSliceTest(unittest.TestCase):
    CUBE_PATH = 'test-cube.zarr'
    CUBE_PATH_2 = 'test-cube-2.zarr'

    def setUp(self) -> None:
        rimraf(self.CUBE_PATH)
        rimraf(self.CUBE_PATH_2)

    def tearDown(self) -> None:
        rimraf(self.CUBE_PATH)
        rimraf(self.CUBE_PATH_2)

    def write_slice(self, start_date):
        self.write_cube(start_date, 1)

    def make_slice(self, start_date) -> xr.Dataset:
        return self.make_cube(start_date, 1)

    def write_cube(self, start_date, num_days: int):
        cube = self.make_cube(start_date, num_days)
        cube.to_zarr(self.CUBE_PATH)

    def make_cube(self, start_date, num_days: int) -> xr.Dataset:
        cube = new_cube(time_periods=num_days,
                        time_freq='1D',
                        time_start=start_date,
                        variables=dict(precipitation=0.1,
                                       temperature=270.5,
                                       soil_moisture=0.2))
        chunk_sizes = dict(time=1, lat=90, lon=90)
        cube = chunk_dataset(cube, chunk_sizes, format_name='zarr')
        return cube

    def test_find_time_slice(self):
        self.write_cube('2019-01-01', 10)

        # Cube does not exists --> write new
        result = find_time_slice(self.CUBE_PATH_2, np.datetime64('2018-12-30T13:00'))
        self.assertEqual((-1, 'create'), result)

        # Before first step --> insert before 0
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2018-12-30T13:00'))
        self.assertEqual((0, 'insert'), result)

        # After first step --> insert before 1
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2019-01-02'))
        self.assertEqual((1, 'insert'), result)

        # In-between --> insert before 5
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2019-01-06T10:00:00'))
        self.assertEqual((5, 'insert'), result)

        # In-between at existing time-stamp --> replace 5
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2019-01-06T12:00:00'))
        self.assertEqual((5, 'replace'), result)

        # After last step --> append
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2019-01-12'))
        self.assertEqual((-1, 'append'), result)

    def test_append_time_slice(self):
        self.write_slice('2019-01-01T14:30')

        append_time_slice(self.CUBE_PATH, self.make_slice('2019-01-02T01:14'))
        append_time_slice(self.CUBE_PATH, self.make_slice('2019-01-02T05:16'))
        append_time_slice(self.CUBE_PATH, self.make_slice('2019-01-03T03:50'))

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(['2019-01-02T02:30',
                             '2019-01-02T13:14',
                             '2019-01-02T17:16',
                             '2019-01-03T15:50'], dtype=cube.time.dtype)
        self.assertEqual(4, cube.time.size)
        self.assertEqual(None, cube.time.chunks)
        np.testing.assert_equal(cube.time.values, expected)

    def test_insert_time_slice(self):
        self.write_cube('2019-01-02T00:00:00', 10)

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(
            ['2019-01-02T12:00:00', '2019-01-03T12:00:00',
             '2019-01-04T12:00:00', '2019-01-05T12:00:00',
             '2019-01-06T12:00:00', '2019-01-07T12:00:00',
             '2019-01-08T12:00:00', '2019-01-09T12:00:00',
             '2019-01-10T12:00:00', '2019-01-11T12:00:00'],
            dtype=cube.time.dtype)
        np.testing.assert_equal(cube.time.values, expected)

        insert_time_slice(self.CUBE_PATH, 5,
                          self.make_slice('2019-01-06T02:00:00'))

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(
            ['2019-01-02T12:00:00', '2019-01-03T12:00:00',
             '2019-01-04T12:00:00', '2019-01-05T12:00:00',
             '2019-01-06T12:00:00', '2019-01-06T14:00:00', '2019-01-07T12:00:00',
             '2019-01-08T12:00:00', '2019-01-09T12:00:00',
             '2019-01-10T12:00:00', '2019-01-11T12:00:00'],
            dtype=cube.time.dtype)
        np.testing.assert_equal(cube.time.values, expected)

        insert_time_slice(self.CUBE_PATH, 10, self.make_slice('2019-01-10T02:00'))
        insert_time_slice(self.CUBE_PATH, 0, self.make_slice('2019-01-01T02:00'))

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(['2019-01-01T14:00', '2019-01-02T12:00',
                             '2019-01-03T12:00', '2019-01-04T12:00',
                             '2019-01-05T12:00', '2019-01-06T12:00',
                             '2019-01-06T14:00', '2019-01-07T12:00',
                             '2019-01-08T12:00', '2019-01-09T12:00',
                             '2019-01-10T12:00', '2019-01-10T14:00',
                             '2019-01-11T12:00'], dtype=cube.time.dtype)
        actual = cube.time.values
        print(actual)
        self.assertEqual(len(expected), len(actual))
        self.assertEqual(None, cube.time.chunks)
        np.testing.assert_equal(actual, expected)

    def test_replace_time_slice(self):
        self.write_cube('2019-01-02', 10)

        replace_time_slice(self.CUBE_PATH, 5, self.make_slice('2019-01-06T02:00'))
        replace_time_slice(self.CUBE_PATH, 9, self.make_slice('2019-01-11T02:00'))
        replace_time_slice(self.CUBE_PATH, 0, self.make_slice('2019-01-01T02:00'))

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(['2019-01-01T14:00', '2019-01-03T12:00',
                             '2019-01-04T12:00', '2019-01-05T12:00',
                             '2019-01-06T12:00', '2019-01-06T14:00',
                             '2019-01-08T12:00', '2019-01-09T12:00',
                             '2019-01-10T12:00', '2019-01-11T14:00'], dtype=cube.time.dtype)
        self.assertEqual(10, cube.time.size)
        self.assertEqual(None, cube.time.chunks)
        np.testing.assert_equal(cube.time.values, expected)

    def test_update_corrupt_cube(self):
        self.write_cube('2019-01-01', 3)

        cube = xr.open_zarr(self.CUBE_PATH)
        t, y, x = cube.precipitation.shape
        new_shape = y, t, x
        t, y, x = cube.precipitation.dims
        new_dims = y, t, x
        cube['precipitation'] = xr.DataArray(cube.precipitation.values.reshape(new_shape),
                                             dims=new_dims,
                                             coords=cube.precipitation.coords)
        cube.to_zarr(self.CUBE_PATH_2)

        with self.assertRaises(ValueError) as cm:
            insert_time_slice(self.CUBE_PATH_2, 2, self.make_slice('2019-01-02T06:30'))
        self.assertEqual("dimension 'time' of variable 'precipitation' must be first dimension",
                         f"{cm.exception}")


class ZarrStoreTest(unittest.TestCase):
    CUBE_PATH = 'store-test-cube.zarr'

    def setUp(self) -> None:
        rimraf(self.CUBE_PATH)

    def tearDown(self) -> None:
        rimraf(self.CUBE_PATH)

    def test_local(self):
        cube = new_cube(time_periods=10, time_start='2019-01-01',
                        variables=dict(precipitation=0.1,
                                       temperature=270.5,
                                       soil_moisture=0.2))
        cube = chunk_dataset(cube, dict(time=1, lat=90, lon=90), format_name='zarr')
        cube.to_zarr(self.CUBE_PATH)
        cube.close()

        diagnostic_store = DiagnosticStore(zarr.DirectoryStore(self.CUBE_PATH),
                                           logging_observer(log_path='local-cube.log'))
        xr.open_zarr(diagnostic_store)

    @unittest.skipUnless(False, 'is enabled')
    def test_remote(self):
        import s3fs
        endpoint_url = "https://s3.eu-central-1.amazonaws.com"
        s3 = s3fs.S3FileSystem(anon=True, client_kwargs=dict(endpoint_url=endpoint_url))
        s3_store = s3fs.S3Map(root="xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr", s3=s3, check=False)
        diagnostic_store = DiagnosticStore(s3_store, logging_observer(log_path='remote-cube.log'))
        xr.open_zarr(diagnostic_store)


# test/core/test_unchunk.py

import os.path
import unittest

from xcube.constants import FORMAT_NAME_ZARR
from xcube.core.chunk import chunk_dataset
from xcube.core.dsio import rimraf
from xcube.core.new import new_cube
from xcube.core.unchunk import unchunk_dataset


class UnchunkDatasetTest(unittest.TestCase):
    TEST_ZARR = 'test.zarr'

    def setUp(self):
        rimraf(self.TEST_ZARR)
        cube = new_cube(variables=dict(A=0.5, B=-1.5))
        cube = chunk_dataset(cube, chunk_sizes=dict(time=1, lat=90, lon=90), format_name=FORMAT_NAME_ZARR)
        cube.to_zarr(self.TEST_ZARR)

        self.chunked_a_files = {'.zarray',
                                '.zattrs',
                                '0.0.0', '0.0.1', '0.0.2', '0.0.3', '0.1.0', '0.1.1', '0.1.2', '0.1.3',
                                '1.0.0', '1.0.1', '1.0.2', '1.0.3', '1.1.0', '1.1.1', '1.1.2', '1.1.3',
                                '2.0.0', '2.0.1', '2.0.2', '2.0.3', '2.1.0', '2.1.1', '2.1.2', '2.1.3',
                                '3.0.0', '3.0.1', '3.0.2', '3.0.3', '3.1.0', '3.1.1', '3.1.2', '3.1.3',
                                '4.0.0', '4.0.1', '4.0.2', '4.0.3', '4.1.0', '4.1.1', '4.1.2', '4.1.3'}
        self.chunked_b_files = self.chunked_a_files
        self.chunked_time_files = {'.zarray', '.zattrs', '0', '1', '2', '3', '4'}
        self.chunked_lat_files = {'.zattrs', '.zarray', '0', '1'}
        self.chunked_lon_files = {'.zattrs', '.zarray', '0', '1', '2', '3'}

    def tearDown(self):
        rimraf(self.TEST_ZARR)

    def test_chunked(self):
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files=self.chunked_b_files,
                                expected_time_files=self.chunked_time_files,
                                expected_lat_files=self.chunked_lat_files,
                                expected_lon_files=self.chunked_lon_files)

    def test_unchunk_all(self):
        unchunk_dataset(self.TEST_ZARR)
        self._assert_cube_files(expected_a_files={'.zarray', '.zattrs', '0.0.0'},
                                expected_b_files={'.zarray', '.zattrs', '0.0.0'},
                                expected_time_files={'.zarray', '.zattrs', '0'},
                                expected_lat_files={'.zarray', '.zattrs', '0'},
                                expected_lon_files={'.zarray', '.zattrs', '0'})

    def test_unchunk_coords_only(self):
        unchunk_dataset(self.TEST_ZARR, coords_only=True)
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files=self.chunked_b_files,
                                expected_time_files={'.zarray', '.zattrs', '0'},
                                expected_lat_files={'.zarray', '.zattrs', '0'},
                                expected_lon_files={'.zarray', '.zattrs', '0'})

    def test_unchunk_data_var(self):
        unchunk_dataset(self.TEST_ZARR, var_names=['B'])
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files={'.zarray', '.zattrs', '0.0.0'},
                                expected_time_files=self.chunked_time_files,
                                expected_lat_files=self.chunked_lat_files,
                                expected_lon_files=self.chunked_lon_files)

    def test_unchunk_coord_var(self):
        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files=self.chunked_b_files,
                                expected_time_files={'.zarray', '.zattrs', '0'},
                                expected_lat_files=self.chunked_lat_files,
                                expected_lon_files=self.chunked_lon_files)

    def test_dont_unchunk_if_unchunked(self):
        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files=self.chunked_b_files,
                                expected_time_files={'.zarray', '.zattrs', '0'},
                                expected_lat_files=self.chunked_lat_files,
                                expected_lon_files=self.chunked_lon_files)

    def test_unchunk_data_var_coords_only(self):
        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR, var_names=['B'], coords_only=True)
        self.assertEqual("variable 'B' is not a coordinate variable in 'test.zarr'", f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR, var_names=['C'], coords_only=False)
        self.assertEqual("variable 'C' is not a variable in 'test.zarr'", f'{cm.exception}')

    def test_unchunk_invalid_path(self):
        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR + '.zip')
        self.assertEqual("'test.zarr.zip' is not a valid Zarr directory", f'{cm.exception}')

    def test_unchunk_invalid_vars(self):
        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR, var_names=['times'], coords_only=True)
        self.assertEqual("variable 'times' is not a coordinate variable in 'test.zarr'", f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR, var_names=['CHL'], coords_only=False)
        self.assertEqual("variable 'CHL' is not a variable in 'test.zarr'", f'{cm.exception}')

    def _assert_cube_files(self,
                           expected_a_files, expected_b_files,
                           expected_time_files, expected_lat_files, expected_lon_files):
        self._assert_files('A', expected_a_files)
        self._assert_files('B', expected_b_files)
        self._assert_files('time', expected_time_files)
        self._assert_files('lat', expected_lat_files)
        self._assert_files('lon', expected_lon_files)

    def _assert_files(self, var_name, expected_files):
        actual_files = os.listdir(os.path.join(self.TEST_ZARR, var_name))
        self.assertEqual(set(expected_files), set(actual_files))


# test/core/test_update.py

import unittest

import numpy as np
import pandas as pd
import xarray as xr

from test.sampledata import create_highroc_dataset
from xcube.core.update import update_dataset_var_attrs, update_dataset_attrs


class UpdateVariablePropsTest(unittest.TestCase):
    def test_no_change(self):
        ds1 = create_highroc_dataset()
        # noinspection PyTypeChecker
        ds2 = update_dataset_var_attrs(ds1, None)
        self.assertIs(ds2, ds1)
        ds2 = update_dataset_var_attrs(ds1, [])
        self.assertIs(ds2, ds1)

    def test_change_all_or_none(self):
        ds1 = create_highroc_dataset()
        ds2 = update_dataset_var_attrs(ds1,
                                       [(var_name, {'marker': True}) for var_name in ds1.data_vars])
        self.assertEqual(len(ds1.data_vars), len(ds2.data_vars))
        self.assertTrue(all(['marker' in ds2[n].attrs for n in ds2.variables]))

        with self.assertRaises(KeyError):
            update_dataset_var_attrs(ds1, [('bibo', {'marker': True})])

    def test_change_some(self):
        ds1 = create_highroc_dataset()
        ds2 = update_dataset_var_attrs(ds1,
                                       [('conc_chl', {'name': 'chl_c2rcc'}),
                                        ('c2rcc_flags', {'name': 'flags', 'marker': True}),
                                        ('rtoa_10', None)])

        self.assertEqual(len(ds1.data_vars), len(ds2.data_vars))

        self.assertNotIn('conc_chl', ds2.data_vars)
        self.assertNotIn('c2rcc_flags', ds2.data_vars)

        self.assertIn('chl_c2rcc', ds2.data_vars)
        self.assertIn('original_name', ds2.chl_c2rcc.attrs)
        self.assertEqual('conc_chl', ds2.chl_c2rcc.attrs['original_name'])

        self.assertIn('flags', ds2.data_vars)
        self.assertIn('original_name', ds2.flags.attrs)
        self.assertEqual('c2rcc_flags', ds2.flags.attrs['original_name'])
        self.assertIn('marker', ds2.flags.attrs)
        self.assertEqual(True, ds2.flags.attrs['marker'])

        self.assertIn('rtoa_10', ds2.data_vars)

        with self.assertRaises(ValueError) as cm:
            update_dataset_var_attrs(ds1, [('conc_chl', None),
                                           ('c2rcc_flags', None),
                                           ('rtoa_1', {'name': 'refl_toa'}),
                                           ('rtoa_2', {'name': 'refl_toa'}),
                                           ('rtoa_3', {'name': 'refl_toa'})])
        self.assertEqual("variable 'rtoa_2' cannot be renamed into 'refl_toa' because the name is already in use",
                         f'{cm.exception}')


class UpdateGlobalAttributesTest(unittest.TestCase):

    def test_update_global_attributes(self):
        num_lons = 8
        num_lats = 6
        num_times = 5

        lon_min = -20.
        lat_min = 12.

        res = 0.25
        res05 = res / 2
        lon = np.linspace(lon_min + res05, lon_min + num_lons * res - res05, num_lons)
        lat = np.linspace(lat_min + res05, lat_min + num_lats * res - res05, num_lats)
        lon_bnds = np.array([[v - res05, v + res05] for v in lon])
        lat_bnds = np.array([[v - res05, v + res05] for v in lat])
        time = [pd.to_datetime(f'2018-06-0{i}T12:00:00') for i in range(1, num_times + 1)]
        time_bnds = [(pd.to_datetime(f'2018-06-0{i}T00:00:00'),
                      pd.to_datetime(f'2018-06-0{i}T23:00:59')) for i in range(1, num_times + 1)]

        coords = dict(time=(['time'], time),
                      lat=(['lat'], lat),
                      lon=(['lon'], lon))

        coords_with_bnds = dict(time_bnds=(['time', 'bnds'], time_bnds),
                                lat_bnds=(['lat', 'bnds'], lat_bnds),
                                lon_bnds=(['lon', 'bnds'], lon_bnds),
                                **coords)

        output_metadata = dict(history='pipo', license='MIT', Conventions='CF-1.7')

        ds1 = xr.Dataset(coords=coords)
        ds2 = update_dataset_attrs(ds1, global_attrs=output_metadata)

        self.assertIsNot(ds2, ds1)
        self.assertEqual('CF-1.7', ds2.attrs.get('Conventions'))
        self.assertEqual('MIT', ds2.attrs.get('license'))
        self.assertEqual('pipo', ds2.attrs.get('history'))
        self.assertEqual(-20.0, ds2.attrs.get('geospatial_lon_min'))
        self.assertEqual(-18.0, ds2.attrs.get('geospatial_lon_max'))
        self.assertEqual(0.25, ds2.attrs.get('geospatial_lon_resolution'))
        self.assertEqual('degrees_east', ds2.attrs.get('geospatial_lon_units'))
        self.assertEqual(12.0, ds2.attrs.get('geospatial_lat_min'))
        self.assertEqual(13.5, ds2.attrs.get('geospatial_lat_max'))
        self.assertEqual(0.25, ds2.attrs.get('geospatial_lat_resolution'))
        self.assertEqual('degrees_north', ds2.attrs.get('geospatial_lat_units'))
        self.assertEqual('2018-06-01T00:00:00.000000000', ds2.attrs.get('time_coverage_start'))
        self.assertEqual('2018-06-06T00:00:00.000000000', ds2.attrs.get('time_coverage_end'))
        self.assertIn('date_modified', ds2.attrs)

        ds1 = xr.Dataset(coords=coords_with_bnds)
        ds2 = update_dataset_attrs(ds1, global_attrs=output_metadata)

        self.assertIsNot(ds2, ds1)
        self.assertEqual('CF-1.7', ds2.attrs.get('Conventions'))
        self.assertEqual('MIT', ds2.attrs.get('license'))
        self.assertEqual('pipo', ds2.attrs.get('history'))
        self.assertEqual(-20.0, ds2.attrs.get('geospatial_lon_min'))
        self.assertEqual(-18.0, ds2.attrs.get('geospatial_lon_max'))
        self.assertEqual(0.25, ds2.attrs.get('geospatial_lon_resolution'))
        self.assertEqual('degrees_east', ds2.attrs.get('geospatial_lon_units'))
        self.assertEqual(12.0, ds2.attrs.get('geospatial_lat_min'))
        self.assertEqual(13.5, ds2.attrs.get('geospatial_lat_max'))
        self.assertEqual(0.25, ds2.attrs.get('geospatial_lat_resolution'))
        self.assertEqual('degrees_north', ds2.attrs.get('geospatial_lat_units'))
        self.assertEqual('2018-06-01T00:00:00.000000000', ds2.attrs.get('time_coverage_start'))
        self.assertEqual('2018-06-05T23:00:59.000000000', ds2.attrs.get('time_coverage_end'))
        self.assertIn('date_modified', ds2.attrs)


# test/core/test_vars2dim.py

import unittest

from xcube.core.new import new_cube
from xcube.core.vars2dim import vars_to_dim


class VarsToDimTest(unittest.TestCase):
    def test_vars_to_dim(self):
        dataset = new_cube(variables=dict(precipitation=0.4, temperature=275.2))

        ds = vars_to_dim(dataset)

        self.assertIn("var", ds.dims)
        self.assertEqual(2, ds.dims["var"])
        self.assertIn("var", ds.coords)
        self.assertIn("data", ds.data_vars)
        var_names = ds["var"]
        self.assertEqual(("var",), var_names.dims)
        self.assertTrue(hasattr(var_names, "encoding"))
        self.assertEqual(2, len(var_names))
        self.assertIn("precipitation", str(var_names[0]))
        self.assertIn("temperature", str(var_names[1]))


# test/core/test_verify.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.new import new_cube
from xcube.core.verify import verify_cube, assert_cube


class AssertAndVerifyCubeTest(unittest.TestCase):

    def test_assert_cube_ok(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        self.assertIs(cube, assert_cube(cube))

    def test_assert_cube_without_bounds(self):
        cube = new_cube(variables=dict(precipitation=0.5), drop_bounds=True)
        self.assertIs(cube, assert_cube(cube))

    def test_assert_cube_illegal_coord_var(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        cube = cube.assign_coords(lat=xr.DataArray(np.outer(cube.lat, np.ones(cube.lon.size)),
                                                   dims=("y", "x")),
                                  lon=xr.DataArray(np.outer(np.ones(cube.lat.size), cube.lon),
                                                   dims=("y", "x")))
        with self.assertRaises(ValueError) as cm:
            assert_cube(cube)
        self.assertEqual("Dataset is not a valid xcube dataset, because:\n"
                         "- missing spatial x,y coordinate variables.",
                         f"{cm.exception}")

    def test_assert_cube_illegal_coord_bounds_var(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        lat_bnds = np.zeros((cube.time.size, cube.lat.size, 2))
        lon_bnds = np.zeros((cube.time.size, cube.lon.size, 2), dtype=np.float16)
        lat_bnds[:, :, :] = cube.lat_bnds
        lon_bnds[:, :, :] = cube.lon_bnds
        cube = cube.assign_coords(lat_bnds=xr.DataArray(lat_bnds, dims=("time", "lat", "bnds")),
                                  lon_bnds=xr.DataArray(lon_bnds, dims=("time", "lon", "bnds")))
        with self.assertRaises(ValueError) as cm:
            assert_cube(cube)
        self.assertEqual("Dataset is not a valid xcube dataset, because:\n"
                         "- bounds coordinate variable 'lon_bnds' must have dimensions ('lon', <bounds_dim>);\n"
                         "- shape of bounds coordinate variable 'lon_bnds' must be (360, 2) but was (5, 360, 2);\n"
                         "- type of bounds coordinate variable 'lon_bnds' must be dtype('float64') but was dtype('float16');\n"
                         "- bounds coordinate variable 'lat_bnds' must have dimensions ('lat', <bounds_dim>);\n"
                         "- shape of bounds coordinate variable 'lat_bnds' must be (180, 2) but was (5, 180, 2).",
                         f"{cm.exception}")

    def test_assert_cube_illegal_data_var(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        shape = cube.dims["lat"], cube.dims["lon"]
        cube["chl"] = xr.DataArray(np.random.rand(*shape),
                                   dims=("lat", "lon"),
                                   coords=dict(lat=cube.lat, lon=cube.lon))
        with self.assertRaises(ValueError) as cm:
            assert_cube(cube)
        self.assertEqual("Dataset is not a valid xcube dataset, because:\n"
                         "- dimensions of data variable 'chl' must be"
                         " ('time', ..., 'lat', 'lon'), but were ('lat', 'lon') for 'chl';\n"
                         "- dimensions of all data variables must be same,"
                         " but found ('time', 'lat', 'lon') for 'precipitation'"
                         " and ('lat', 'lon') for 'chl'.",
                         f"{cm.exception}")

    def test_verify_cube(self):
        cube = new_cube()
        self.assertEqual([], verify_cube(cube))
        ds = cube.drop("time")
        self.assertEqual(["missing time coordinate variable"], verify_cube(ds))
        ds = ds.drop("lat")
        self.assertEqual(["missing spatial x,y coordinate variables",
                          "missing time coordinate variable"], verify_cube(ds))

    def test_verify_cube_coord_equidistance(self):
        ds = new_cube()
        ds['lat'] = ds['lat'] + np.random.rand(len(ds['lat']))
        ds['lat_bnds'][:, 0] = ds['lat_bnds'][:, 0] + np.random.rand(len(ds['lat_bnds'][:, 0]))
        ds['lat_bnds'][:, 1] = ds['lat_bnds'][:, 1] + np.random.rand(len(ds['lat_bnds'][:, 1]))

        self.assertEqual(["coordinate variable 'lat' is not equidistant",
                          "coordinate variable 'lat_bnds' is not equidistant"],
                         verify_cube(ds))

    def test_verify_cube_coord_equidistance_reverse_lat(self):
        ds = new_cube()
        ds['lat'] = np.flip(ds.lat)
        ds['lat_bnds'] = xr.DataArray(np.flip(ds.lat_bnds.values),
                                      dims=['lat', 'bnds'])

        ds['lon'] = np.roll(ds.lon, 90)
        ds['lon_bnds'] = xr.DataArray(np.roll(ds.lon_bnds.values, 180),
                                      dims=['lon', 'bnds'])

        self.assertEqual([], verify_cube(ds))


# test/core/test_xarray.py

import unittest

import xarray as xr

from test.sampledata import new_test_dataset
from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.xarray import DatasetAccessor


class XCubeDatasetAccessorTest(unittest.TestCase):
    # noinspection PyMethodMayBeStatic
    def test_init(self):
        DatasetAccessor(xr.Dataset())

    def test_installed(self):
        self.assertTrue(hasattr(xr.Dataset, "xcube"))
        ds = xr.Dataset()
        self.assertTrue(hasattr(ds, "xcube"))

    def test_non_empty_cube_subset(self):
        dataset = new_cube(variables=dict(a=9, b=0.2))
        cube = dataset.xcube.cube
        self.assertIsInstance(cube, xr.Dataset)
        self.assertEqual(set(dataset.data_vars), set(cube.data_vars))
        gm = dataset.xcube.gm
        self.assertIsInstance(gm, GridMapping)

    def test_no_cube_subset(self):
        dataset = xr.Dataset(dict(a=9, b=0.2))
        cube = dataset.xcube.cube
        self.assertIsInstance(cube, xr.Dataset)
        self.assertEqual(set(), set(cube.data_vars))
        self.assertIs(None, dataset.xcube.gm)
        self.assertIs(dataset, dataset.xcube.non_cube)

    ########################################################################
    # Testing old API from here on.
    #
    # Let's quickly agree, if we should deprecate all this stuff. I guess,
    # no one uses it.
    #
    # We should only add props and methods to this accessor
    # that require a certain state to be hold.
    # Such state could be props that are expensive to recompute,
    # such as grid mappings.
    #
    # It causes too much overhead and maintenance work
    # if we continue putting any xcube function here.
    ########################################################################

    def test_new(self):
        self.assertIsInstance(xr.Dataset.xcube.new(), xr.Dataset)

    def test_select_vars(self):
        self.assertIsInstance(xr.Dataset().xcube.select_variables_subset(), xr.Dataset)

    def test_dump(self):
        self.assertIsInstance(xr.Dataset().xcube.dump(), str)

    def test_vars_to_dim(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05"],
                                   precipitation=0.4, temperature=275.2)

        self.assertIsInstance(dataset.xcube.vars_to_dim(), xr.Dataset)

    def test_levels(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05"],
                                   precipitation=0.4, temperature=275.2)
        levels = dataset.xcube.levels(spatial_tile_shape=(45, 45))
        self.assertIsInstance(levels, list)
        self.assertEqual(3, len(levels))
        self.assertTrue(all(isinstance(level, xr.Dataset) for level in levels))
        self.assertTrue(all("precipitation" in level for level in levels))
        self.assertTrue(all("temperature" in level for level in levels))
        self.assertEqual([(5, 180, 360), (5, 90, 180), (5, 45, 90)],
                         [level.precipitation.shape for level in levels])
        self.assertEqual([((1, 1, 1, 1, 1), (45, 45, 45, 45), (45, 45, 45, 45, 45, 45, 45, 45)),
                          ((1, 1, 1, 1, 1), (45, 45), (45, 45, 45, 45)),
                          ((1, 1, 1, 1, 1), (45,), (45, 45))],
                         [level.precipitation.chunks for level in levels])


# test/util/test_assert.py

from unittest import TestCase

from xcube.util.assertions import assert_true
from xcube.util.assertions import assert_false
from xcube.util.assertions import assert_condition
from xcube.util.assertions import assert_not_none
from xcube.util.assertions import assert_given
from xcube.util.assertions import assert_instance
from xcube.util.assertions import assert_in


class AssertTest(TestCase):
    def test_assert_not_none(self):
        assert_not_none(10, 'x')
        with self.assertRaises(ValueError) as e:
            assert_not_none(None, 'x')
        self.assertEqual(('x must not be None',), e.exception.args)

    def test_assert_given(self):
        assert_given('data.txt', 'x')
        with self.assertRaises(ValueError) as e:
            assert_given('', 'x')
        self.assertEqual(('x must be given',), e.exception.args)

    def test_assert_instance(self):
        assert_instance(10, int, 'x')
        assert_instance('data.txt', (int, str), 'x')
        with self.assertRaises(TypeError) as e:
            assert_instance(0.5, (int, str), 'x')
        self.assertEqual(("x must be an instance of "
                          "(<class 'int'>, <class 'str'>), "
                          "was <class 'float'>",),
                         e.exception.args)

    def test_assert_in(self):
        assert_in(2, (1, 2, 3), 'x')
        with self.assertRaises(ValueError) as e:
            assert_in(4, (1, 2, 3), 'x')
        self.assertEqual(('x must be one of (1, 2, 3)',), e.exception.args)

    def test_assert_true(self):
        assert_true(True, 'Should be true')
        with self.assertRaises(ValueError) as e:
            assert_true(False, 'Should be true')
        self.assertEqual(('Should be true',), e.exception.args)

    def test_assert_false(self):
        assert_false(False, 'Should be false')
        with self.assertRaises(ValueError) as e:
            assert_false(True, 'Should be false')
        self.assertEqual(('Should be false',), e.exception.args)

    def test_assert_condition(self):
        assert_condition(True, 'Should be true')
        with self.assertRaises(ValueError) as e:
            assert_condition(False, 'Should be true')
        self.assertEqual(('Should be true',), e.exception.args)


# test/util/test_cache.py

import os
import shutil
import unittest

from xcube.util.cache import CacheStore, Cache, MemoryCacheStore, FileCacheStore, parse_mem_size


class MemoryCacheStoreTest(unittest.TestCase):
    def setUp(self):
        self.cache_store = MemoryCacheStore()
        stored_value_a, size_a = self.cache_store.store_value('a', 42)
        stored_value_b, size_b = self.cache_store.store_value('b', True)
        stored_value_c, size_c = self.cache_store.store_value('c', "S" * 256)
        stored_value_d, size_d = self.cache_store.store_value('d', 3.14)
        self.stored_value_a = stored_value_a
        self.stored_value_b = stored_value_b
        self.stored_value_c = stored_value_c
        self.stored_value_d = stored_value_d
        self.assertEqual(size_a, 28)
        self.assertEqual(size_b, 28)
        self.assertEqual(size_c, 305)
        self.assertEqual(size_d, 24)

    def test_store_value(self):
        self.assertEqual(self.stored_value_a, ['a', 42])
        self.assertEqual(self.stored_value_b, ['b', True])
        self.assertEqual(self.stored_value_c, ['c', "S" * 256])
        self.assertEqual(self.stored_value_d, ['d', 3.14])

    def test_restore_value(self):
        self.assertEqual(self.cache_store.restore_value('a', self.stored_value_a), 42)
        self.assertEqual(self.cache_store.restore_value('b', self.stored_value_b), True)
        with self.assertRaises(ValueError):
            self.cache_store.restore_value('e', self.stored_value_b)

    def test_discard_value(self):
        self.cache_store.discard_value('a', self.stored_value_a)
        self.cache_store.discard_value('b', self.stored_value_b)
        self.assertEqual(self.cache_store.restore_value('a', self.stored_value_a), None)
        self.assertEqual(self.cache_store.restore_value('b', self.stored_value_b), None)
        with self.assertRaises(ValueError):
            self.cache_store.discard_value('e', self.stored_value_b)


class FileCacheStoreTest(unittest.TestCase):
    DIR = '__test_file_cache__'

    def setUp(self):
        if os.path.exists(FileCacheStoreTest.DIR):
            shutil.rmtree(FileCacheStoreTest.DIR, ignore_errors=True)

        os.mkdir(FileCacheStoreTest.DIR)
        try:
            self.cache_store = FileCacheStore(FileCacheStoreTest.DIR, ".dat")
            stored_value_a, size_a = self.cache_store.store_value('a', bytes('abc', 'utf8'))
            stored_value_b, size_b = self.cache_store.store_value('b', bytes('def', 'utf8'))
            stored_value_c, size_c = self.cache_store.store_value('c', bytes('ghi', 'utf8'))
            self.stored_value_a = stored_value_a
            self.stored_value_b = stored_value_b
            self.stored_value_c = stored_value_c
            self.assertEqual(size_a, 3)
            self.assertEqual(size_b, 3)
            self.assertEqual(size_c, 3)
        except Exception as e:
            shutil.rmtree(FileCacheStoreTest.DIR, ignore_errors=True)
            raise e

    def tearDown(self):
        shutil.rmtree(FileCacheStoreTest.DIR, ignore_errors=True)

    def test_store_value(self):
        self.assertEqual(self.stored_value_a, os.path.join(FileCacheStoreTest.DIR, 'a.dat'))
        self.assertEqual(self.stored_value_b, os.path.join(FileCacheStoreTest.DIR, 'b.dat'))
        self.assertEqual(self.stored_value_c, os.path.join(FileCacheStoreTest.DIR, 'c.dat'))

    def test_restore_value(self):
        self.assertEqual(self.cache_store.restore_value('a', self.stored_value_a), bytes('abc', 'utf8'))
        self.assertEqual(self.cache_store.restore_value('b', self.stored_value_b), bytes('def', 'utf8'))
        self.assertEqual(self.cache_store.restore_value('c', self.stored_value_c), bytes('ghi', 'utf8'))
        with self.assertRaises(FileNotFoundError):
            self.cache_store.restore_value('e', self.stored_value_b)

    def test_discard_value(self):
        self.cache_store.discard_value('a', self.stored_value_a)
        self.cache_store.discard_value('b', self.stored_value_b)
        self.cache_store.discard_value('c', self.stored_value_c)
        self.cache_store.discard_value('e', self.stored_value_b)
        with self.assertRaises(FileNotFoundError):
            self.cache_store.restore_value('a', self.stored_value_a)
        with self.assertRaises(FileNotFoundError):
            self.cache_store.restore_value('b', self.stored_value_b)
        with self.assertRaises(FileNotFoundError):
            self.cache_store.restore_value('c', self.stored_value_c)


class TracingCacheStore(CacheStore):
    def __init__(self):
        self.trace = ''

    def can_load_from_key(self, key) -> bool:
        self.trace += 'can_load_from_key(%s);' % key
        return key == 'k5'

    def load_from_key(self, key):
        self.trace += 'load_from_key(%s);' % key
        if key == 'k5':
            return 'S/yyyy', 600
        raise ValueError()

    def store_value(self, key, value):
        self.trace += 'store(%s, %s);' % (key, value)
        return 'S/' + value, 100 * len(value)

    def restore_value(self, key, stored_value):
        self.trace += 'restore(%s, %s);' % (key, stored_value)
        return stored_value[2:]

    def discard_value(self, key, stored_value):
        self.trace += 'discard(%s, %s);' % (key, stored_value)


class CacheTest(unittest.TestCase):
    def setUp(self):
        pass

    def test_store_and_restore_and_discard(self):
        cache_store = TracingCacheStore()
        cache = Cache(store=cache_store, capacity=1000)

        self.assertIs(cache.store, cache_store)
        self.assertEqual(cache.size, 0)
        self.assertEqual(cache.max_size, 750)

        cache_store.trace = ''
        cache.put_value('k1', 'x')
        self.assertEqual(cache.get_value('k1'), 'x')
        self.assertEqual(cache.size, 100)
        self.assertEqual(cache_store.trace, 'store(k1, x);restore(k1, S/x);')

        cache_store.trace = ''
        cache.remove_value('k1')
        self.assertEqual(cache.size, 0)
        self.assertEqual(cache_store.trace, 'discard(k1, S/x);')
        cache_store.trace = ''

        cache_store.trace = ''
        cache.put_value('k1', 'x')
        cache.put_value('k1', 'xx')
        self.assertEqual(cache.get_value('k1'), 'xx')
        self.assertEqual(cache.size, 200)
        self.assertEqual(cache_store.trace, 'store(k1, x);discard(k1, S/x);store(k1, xx);restore(k1, S/xx);')

        cache_store.trace = ''
        cache.remove_value('k1')
        self.assertEqual(cache.size, 0)
        self.assertEqual(cache_store.trace, 'discard(k1, S/xx);')

        cache_store.trace = ''
        cache.put_value('k1', 'x')
        cache.put_value('k2', 'xxx')
        cache.put_value('k3', 'xx')
        self.assertEqual(cache.get_value('k1'), 'x')
        self.assertEqual(cache.get_value('k2'), 'xxx')
        self.assertEqual(cache.get_value('k3'), 'xx')
        self.assertEqual(cache.size, 600)
        self.assertEqual(cache_store.trace, 'store(k1, x);store(k2, xxx);store(k3, xx);'
                                            'restore(k1, S/x);restore(k2, S/xxx);restore(k3, S/xx);')

        cache_store.trace = ''
        cache.put_value('k4', 'xxxx')
        self.assertEqual(cache.size, 600)
        self.assertEqual(cache_store.trace, 'store(k4, xxxx);discard(k1, S/x);discard(k2, S/xxx);')

        cache_store.trace = ''
        cache.clear()
        self.assertEqual(cache.size, 0)

    def test_load_from_key(self):
        cache_store = TracingCacheStore()
        cache = Cache(store=cache_store, capacity=1000)

        cache_store.trace = ''
        self.assertEqual(cache.get_value('k1'), None)
        self.assertEqual(cache.size, 0)
        self.assertEqual(cache_store.trace, 'can_load_from_key(k1);')

        cache_store.trace = ''
        self.assertEqual(cache.get_value('k5'), 'yyyy')
        self.assertEqual(cache.size, 600)
        self.assertEqual(cache_store.trace, 'can_load_from_key(k5);load_from_key(k5);restore(k5, S/yyyy);')


class ParseMemSizeTest(unittest.TestCase):
    def test_parse_mem_size(self):
        self.assertEqual(None, parse_mem_size(""))
        self.assertEqual(None, parse_mem_size("0"))
        self.assertEqual(None, parse_mem_size("0M"))
        self.assertEqual(None, parse_mem_size("off"))
        self.assertEqual(None, parse_mem_size("OFF"))
        self.assertEqual(None, parse_mem_size("None"))
        self.assertEqual(None, parse_mem_size("null"))
        self.assertEqual(None, parse_mem_size("False"))
        self.assertEqual(200001, parse_mem_size("200001"))
        self.assertEqual(200001, parse_mem_size("200001B"))
        self.assertEqual(300000, parse_mem_size("300K"))
        self.assertEqual(3000000, parse_mem_size("3M"))
        self.assertEqual(7000000, parse_mem_size("7m"))
        self.assertEqual(2000000000, parse_mem_size("2g"))
        self.assertEqual(2000000000, parse_mem_size("2G"))
        self.assertEqual(1000000000000, parse_mem_size("1T"))

        with self.assertRaises(ValueError) as cm:
            parse_mem_size("7n")
        self.assertEqual("invalid memory size: '7N'", f"{cm.exception}")

        with self.assertRaises(ValueError) as cm:
            parse_mem_size("-2g")
        self.assertEqual("negative memory size: '-2G'", f"{cm.exception}")


# test/util/test_caseless.py

import unittest

from xcube.util.caseless import caseless_dict


class CaselessDictTest(unittest.TestCase):
    def test_contains(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        self.assertIn('time', d)
        self.assertIn('TIME', d)
        self.assertIn('lon', d)
        self.assertIn('LON', d)
        self.assertNotIn('lat', d)
        self.assertNotIn('LAT', d)

    def test_getitem(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        self.assertEqual('2010-04-09', d['time'])
        self.assertEqual('2010-04-09', d['TIME'])
        self.assertEqual(53.6, d['lon'])
        self.assertEqual(53.6, d['LON'])

        with self.assertRaises(KeyError):
            # noinspection PyUnusedLocal
            var = d['lat']

    def test_setitem(self):
        d = caseless_dict()
        d['TIME'] = '2010-04-09'
        d['LON'] = 53.6
        self.assertIn('time', d)
        self.assertIn('TIME', d)
        self.assertEqual('2010-04-09', d['time'])
        self.assertIn('lon', d)
        self.assertIn('LON', d)
        self.assertEqual(53.6, d['lon'])
        self.assertNotIn('lat', d)
        self.assertNotIn('LAT', d)

    def test_delitem(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        del d['TIME']
        self.assertNotIn('time', d)
        self.assertNotIn('TIME', d)
        self.assertIn('lon', d)
        self.assertIn('LON', d)
        del d['lon']
        self.assertNotIn('lon', d)
        self.assertNotIn('LON', d)

    def test_get(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        self.assertEqual('2010-04-09', d.get('time'))
        self.assertEqual('2010-04-09', d.get('TIME'))
        self.assertEqual(53.6, d.get('lon'))
        self.assertEqual(53.6, d.get('LON'))
        self.assertEqual(None, d.get('lat'))
        self.assertEqual(-3.1, d.get('LAT', -3.1))

    def test_pop(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        v = d.pop('TIME')
        self.assertEqual('2010-04-09', v)
        self.assertNotIn('time', d)
        self.assertNotIn('TIME', d)
        self.assertIn('lon', d)
        self.assertIn('LON', d)
        v = d.pop('lon')
        self.assertEqual(53.6, v)
        self.assertNotIn('lon', d)
        self.assertNotIn('LON', d)


# test/util/test_cmaps.py

import os
from unittest import TestCase

import matplotlib.cm as cm
from matplotlib.colors import LinearSegmentedColormap, ListedColormap, Colormap

from xcube.util.cmaps import get_cmaps, ensure_cmaps_loaded, _get_custom_colormap, get_cmap


class CmapsTest(TestCase):

    def test_get_cmap(self):
        ensure_cmaps_loaded()

        cmap_name, cmap = get_cmap('plasma')
        self.assertEqual('plasma', cmap_name)
        self.assertIsInstance(cmap, Colormap)

        cmap_name, cmap = get_cmap('PLASMA')
        self.assertEqual('viridis', cmap_name)
        self.assertIsInstance(cmap, Colormap)

        cmap_name, cmap = get_cmap('PLASMA', default_cmap_name='magma')
        self.assertEqual('magma', cmap_name)
        self.assertIsInstance(cmap, Colormap)

        with self.assertRaises(ValueError):
            get_cmap('PLASMA', default_cmap_name='MAGMA')

    def test_get_cmaps_returns_singleton(self):
        cmaps = get_cmaps()
        self.assertIs(cmaps, get_cmaps())
        self.assertIs(cmaps, get_cmaps())

    def test_get_cmaps_registers_ocean_colour(self):
        ensure_cmaps_loaded()
        cmap = cm.get_cmap('deep', 256)
        self.assertTrue((type(cmap) is LinearSegmentedColormap) or (type(cmap) is ListedColormap))

    def test_get_cmaps_retruns_equal_size_recs(self):
        cmaps = get_cmaps()
        rec_len = len(cmaps[0])
        self.assertEqual(rec_len, 3)
        for cmap in cmaps:
            self.assertEqual(len(cmap), rec_len)

    def test_get_cmaps_categories(self):
        cmaps = get_cmaps()
        self.assertGreaterEqual(len(cmaps), 8)
        self.assertEqual(cmaps[0][0], 'Perceptually Uniform Sequential')
        self.assertEqual(cmaps[1][0], 'Sequential 1')
        self.assertEqual(cmaps[2][0], 'Sequential 2')
        self.assertEqual(cmaps[3][0], 'Diverging')
        self.assertEqual(cmaps[4][0], 'Qualitative')
        self.assertEqual(cmaps[5][0], 'Ocean')
        self.assertEqual(cmaps[6][0], 'Miscellaneous')
        self.assertEqual(cmaps[7][0], 'Custom SNAP Colormaps')

    def test_get_cmaps_category_descr(self):
        cmaps = get_cmaps()
        self.assertEqual(cmaps[0][1], 'For many applications, a perceptually uniform colormap is the best choice - '
                                      'one in which equal steps in data are perceived as equal steps in the color '
                                      'space')

    def test_get_cmaps_category_tuples(self):
        cmaps = get_cmaps()
        category_tuple = cmaps[0][2]
        self.assertEqual(len(category_tuple), 8)
        self.assertEqual(category_tuple[0][0], 'viridis')
        self.assertEqual(category_tuple[0][1],
                         'iVBORw0KGgoAAAANSUhEUgAAAQAAAAACCAYAAAC3zQLZAAAAzklEQVR4nO2TQZLFIAhEX7dXmyPM/Y8SZwEqMcnU3/9QZTU8GszC6Ee/HQlk5FAsJIENqVGv/piZ3uqf3nX6Vtd+l8D8UwNOLhZL3+BLh796OXvMdWaqtrrqnZ/tjvuZT/0XxnN/5f25z9X7tIMTKzV7/5yrME3NHoPlUzvplgOevOcz6ZO5eCqzOmark1nHDQveHuuYaazZkTcdmE110HJu6doR3tgfPHyL51zNc0fd2xjf0vPukUPL36YBTcpcWArFyY0RTca88cYbXxt/gUOJC8yRF1kAAAAASUVORK5CYII=')

        self.assertEqual(category_tuple[1][0], 'viridis_alpha')
        self.assertEqual(category_tuple[2][0], 'inferno')
        self.assertEqual(category_tuple[3][0], 'inferno_alpha')

    def test_cmocean_category(self):
        cmaps = get_cmaps()
        category_tuple = cmaps[5][2]
        self.assertTrue(len(category_tuple) >= 34)
        self.assertEqual(category_tuple[0][0], 'thermal')
        self.assertEqual(category_tuple[0][1],
                         'iVBORw0KGgoAAAANSUhEUgAAAQAAAAACCAYAAAC3zQLZAAAA2klEQVR4nO2S6xHDMAiDP+FROkL3Xy30RwBju52g8V0OIcnyKxqvtwsD5SfAUPZNE6M4VR2hJTdQeBX6UhlY8xgDY8V24A15pMuIXcQHJo4qwOQYIHlojpT6zWnzqDxRo+/+zFZbR7H2Tx3WvMPf1qDvq+17zz/m7TV97YxHbefEW27ve+7Oe9xZZu3cdXCdr17XokurfvYOcmTXxHJkE2P32ei8eVxww1WJecRlBxZr/cndj+T5MKULbzqm5pnY56MFjnkmPH7cb7xXzvR49RRO3njGM57xt+MDC391Pt11tkYAAAAASUVORK5CYII=')

        self.assertEqual(category_tuple[1][0], 'thermal_alpha')
        self.assertEqual(category_tuple[2][0], 'haline')
        self.assertEqual(category_tuple[3][0], 'haline_alpha')

    def test_get_cmaps_registers_snap_color(self):
        ensure_cmaps_loaded()
        cmap_name = os.path.join(os.path.dirname(__file__), 'chl_DeM2_200.cpd')
        cmap = _get_custom_colormap(cmap_name)
        cm.register_cmap(cmap=cmap)
        self.assertTrue((type(cmap) is LinearSegmentedColormap) or (type(cmap) is ListedColormap))

    def test_get_cmaps_registers_ivalid_snap_color(self):
        ensure_cmaps_loaded()
        cmap_name = os.path.join(os.path.dirname(__file__), 'chl_DeM2_200_invalid_for_testing.cpd')
        with self.assertRaises(ValueError):
            cmap = _get_custom_colormap(cmap_name)
            cm.register_cmap(cmap=cmap)

    def test_get_cmaps_registers_nonexisting_snap_color(self):
        ensure_cmaps_loaded()
        cmap_name = 'test/webapi/im/chl_DeM2_200_not_existing.cpd'
        with self.assertRaises(ValueError):
            cmap = _get_custom_colormap(cmap_name)
            cm.register_cmap(cmap=cmap)


def main():
    cmaps = get_cmaps()

    html_head = '<!DOCTYPE html>\n' + \
                '<html lang="en">\n' + \
                '<head>' + \
                '<meta charset="UTF-8">' + \
                '<title>matplotlib Color Maps</title>' + \
                '</head>\n' + \
                '<body style="padding: 0.2em">\n'

    html_body = ''

    html_foot = '</body>\n' + \
                '</html>\n'

    for cmap_cat, cmap_desc, cmap_bars in cmaps:
        html_body += '    <h2>%s</h2>\n' % cmap_cat
        html_body += '    <p><i>%s</i></p>\n' % cmap_desc
        html_body += '    <table style=border: 0">\n'
        for cmap_bar in cmap_bars:
            cmap_name, cmap_data = cmap_bar
            cmap_image = '<img src="data:image/png;base64,%s" width="100%%" height="20px"/>' % cmap_data

            html_body += '        <tr><td style="width: 5em">%s:</td><td style="width: 40em">%s</td></tr>\n' % (
                cmap_name, cmap_image)
        html_body += '    </table>\n'

    html_page = html_head + html_body + html_foot

    html_filename = 'test_cmaps.html'
    with open(html_filename, 'w') as fp:
        fp.write(html_page)

    import webbrowser
    webbrowser.open_new_tab(html_filename)


if __name__ == '__main__':
    main()


# test/util/test_config.py

import unittest
from io import StringIO

import yaml
from xcube.util.config import flatten_dict, to_name_dict_pair, to_name_dict_pairs, to_resolved_name_dict_pairs, \
    merge_config


# from yaml.parser import ParserError # needs to be kept for last test, which is still not working properly


class ToResolvedNameDictPairsTest(unittest.TestCase):
    def test_to_resolved_name_dict_pairs_and_reject(self):
        container = ['a', 'a1', 'a2', 'b1', 'b2', 'x_c', 'y_c', 'd', 'e', 'e1', 'e_1', 'e_12', 'e_AB']
        resolved = to_resolved_name_dict_pairs([('a*', None),
                                                ('b', {'name': 'B'}),
                                                ('*_c', {'marker': True, 'name': 'C'}),
                                                ('d', {'name': 'D'}),
                                                ('e_??', {'marker': True})],
                                               container)
        self.assertEqual([('a', None),
                          ('a1', None),
                          ('a2', None),
                          # ('b', {'name': 'B'}),  # 'b' is rejected!
                          ('x_c', {'marker': True, 'name': 'C'}),
                          ('y_c', {'marker': True, 'name': 'C'}),
                          ('d', {'name': 'D'}),
                          ('e_12', {'marker': True}),
                          ('e_AB', {'marker': True})],
                         resolved)

    def test_to_resolved_name_dict_pairs_and_keep(self):
        container = ['a', 'a1', 'a2', 'b1', 'b2', 'x_c', 'y_c', 'd', 'e', 'e1', 'e_1', 'e_12', 'e_AB']
        resolved = to_resolved_name_dict_pairs([('a*', None),
                                                ('b', {'name': 'B'}),
                                                ('*_c', {'marker': True, 'name': 'C'}),
                                                ('d', {'name': 'D'}),
                                                ('e_??', {'marker': True})],
                                               container,
                                               keep=True)
        self.assertEqual([('a', None),
                          ('a1', None),
                          ('a2', None),
                          ('b', {'name': 'B'}),  # 'b' is kept!
                          ('x_c', {'marker': True, 'name': 'C'}),
                          ('y_c', {'marker': True, 'name': 'C'}),
                          ('d', {'name': 'D'}),
                          ('e_12', {'marker': True}),
                          ('e_AB', {'marker': True})],
                         resolved)


class ToNameDictPairsTest(unittest.TestCase):
    def test_to_name_dict_pairs_from_list(self):
        parent = ['a*',
                  ('b', 'B'),
                  ('*_c', {'marker': True, 'name': 'C'}),
                  {'d': 'D'},
                  {'e_??': {'marker': True}}]
        pairs = to_name_dict_pairs(parent, default_key='name')
        self.assertEqual([('a*', None),
                          ('b', {'name': 'B'}),
                          ('*_c', {'marker': True, 'name': 'C'}),
                          ('d', {'name': 'D'}),
                          ('e_??', {'marker': True})],
                         pairs)

    def test_to_name_dict_pairs_from_dict(self):
        parent = {'a*': None,
                  'b': 'B',
                  '*_c': {'marker': True, 'name': 'C'},
                  'd': 'D',
                  'e_??': {'marker': True}}
        pairs = to_name_dict_pairs(parent, default_key='name')
        self.assertEqual([('a*', None),
                          ('b', {'name': 'B'}),
                          ('*_c', {'marker': True, 'name': 'C'}),
                          ('d', {'name': 'D'}),
                          ('e_??', {'marker': True})],
                         pairs)


class ToNameDictPairTest(unittest.TestCase):
    def test_name_only(self):
        pair = to_name_dict_pair('a', default_key='name')
        self.assertEqual(('a', None), pair)
        pair = to_name_dict_pair('a?', default_key='name')
        self.assertEqual(('a?', None), pair)

        with self.assertRaises(ValueError) as cm:
            to_name_dict_pair(12, default_key='name')
        self.assertEqual('name must be a string', f'{cm.exception}')

    def test_tuple(self):
        pair = to_name_dict_pair(('a', 'udu'), default_key='name')
        self.assertEqual(('a', dict(name='udu')), pair)
        pair = to_name_dict_pair(('a*', 'udu'), default_key='name')
        self.assertEqual(('a*', dict(name='udu')), pair)
        pair = to_name_dict_pair(('a*', {'name': 'udu'}))
        self.assertEqual(('a*', dict(name='udu')), pair)
        pair = to_name_dict_pair(('a*', {'marker': True, 'name': 'udu'}))
        self.assertEqual(('a*', dict(marker=True, name='udu')), pair)

        with self.assertRaises(ValueError) as cm:
            to_name_dict_pair(('a', 5))
        self.assertEqual("value of 'a' must be a dictionary", f'{cm.exception}')

    def test_dict(self):
        pair = to_name_dict_pair('a', parent=dict(a='udu'), default_key='name')
        self.assertEqual(('a', dict(name='udu')), pair)
        pair = to_name_dict_pair('a*', parent={'a*': 'udu'}, default_key='name')
        self.assertEqual(('a*', dict(name='udu')), pair)

    def test_mapping(self):
        pair = to_name_dict_pair({'a': 'udu'}, default_key='name')
        self.assertEqual(('a', dict(name='udu')), pair)
        pair = to_name_dict_pair({'a*': dict(name='udu')})
        self.assertEqual(('a*', dict(name='udu')), pair)


class FlattenDictTest(unittest.TestCase):

    def test_flatten_dict(self):
        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            flatten_dict(None)

        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            flatten_dict(673)

        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            flatten_dict("?")

        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            flatten_dict({0: ""})

        self.assertEqual({}, flatten_dict({}))

        self.assertEqual({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                          "source": "Sentinel-3 OLCI L2 surface observation"
                          },
                         flatten_dict({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                                       "source": "Sentinel-3 OLCI L2 surface observation"
                                       }))

        self.assertEqual({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                          "date_created": "2018-05-30",
                          "date_modified": "2018-05-30",
                          "date_issued": "2018-06-01"
                          },
                         flatten_dict({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                                       "date": {
                                           "created": "2018-05-30",
                                           "modified": "2018-05-30",
                                           "issued": "2018-06-01"
                                       }}))

        self.assertEqual({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                          "creator_name": "BC",
                          "creator_url": "http://www.bc.de"
                          },
                         flatten_dict({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                                       "creator": [
                                           {"name": "BC",
                                            "url": "http://www.bc.de"
                                            }
                                       ]}))

        self.assertEqual({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                          "creator_name": "BC, ACME",
                          "creator_url": "http://www.bc.de, http://acme.com"
                          },
                         flatten_dict({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                                       "creator": [
                                           {"name": "BC",
                                            "url": "http://www.bc.de"
                                            },
                                           {"name": "ACME",
                                            "url": "http://acme.com"
                                            }
                                       ]}))

    def test_from_yaml(self):
        stream = StringIO(TEST_JSON)
        d = yaml.full_load(stream)
        d = flatten_dict(d['output_metadata'])
        self.assertEqual(17, len(d))
        self.assertEqual('DCS4COP Sentinel-3 OLCI L2C Data Cube',
                         d.get('title'))
        self.assertEqual('Brockmann Consult GmbH, Royal Belgian Institute for Natural Sciences (RBINS)',
                         d.get('creator_name'))
        self.assertEqual('https://www.brockmann-consult.de, http://odnature.naturalsciences.be/remsem/',
                         d.get('creator_url'))
        self.assertEqual("2018-05-30", d.get('date_created'))
        self.assertEqual("2018-06-01", d.get('date_issued'))
        self.assertEqual(0.0, d.get('geospatial_lon_min'))
        self.assertEqual(5.0, d.get('geospatial_lon_max'))
        self.assertEqual(50.0, d.get('geospatial_lat_min'))
        self.assertEqual(52.5, d.get('geospatial_lat_max'))
        self.assertEqual('degrees_east', d.get('geospatial_lon_units'))
        self.assertEqual('degrees_north', d.get('geospatial_lat_units'))
        self.assertEqual(0.0025, d.get('geospatial_lon_resolution'))
        self.assertEqual(0.0025, d.get('geospatial_lat_resolution'))
        self.assertEqual('2016-10-01', d.get('time_coverage_start'))
        self.assertEqual('2017-10-01T12:00:10', d.get('time_coverage_end'))
        self.assertEqual('P1Y', d.get('time_coverage_duration'))
        self.assertEqual('1D', d.get('time_coverage_resolution'))


TEST_JSON = """
output_metadata:
  # CF: A succinct description of what is in the dataset.
  title: "DCS4COP Sentinel-3 OLCI L2C Data Cube"
  
  # The data creator's name, URL, and email.
  # The "institution" attribute will be used if the "creator_name" attribute does not exist.
  creator:
    - name: "Brockmann Consult GmbH"
      url: "https://www.brockmann-consult.de"
    - name: "Royal Belgian Institute for Natural Sciences (RBINS)"
      url: "http://odnature.naturalsciences.be/remsem/"
  
  date:
    # The date on which the data was created.
    created:  "2018-05-30"
    # The date on which this data was formally issued.
    issued:   "2018-06-01"
  
  geospatial_lon:
    min:  0.0
    max:  5.0
    units: "degrees_east"
    resolution: 0.0025
  
  geospatial_lat:
    min: 50.0
    max: 52.5
    units: "degrees_north"
    resolution: 0.0025
  
  time_coverage:
    start:      2016-10-01
    end:        2017-10-01T12:00:10
    duration:   "P1Y"
    resolution: "1D"
"""


class MergeDictsTest(unittest.TestCase):
    def test_single_dict(self):
        first_dict = {}
        actual_dict = merge_config(first_dict)
        self.assertIs(first_dict, actual_dict)

    def test_two_dicts(self):
        first_dict = {'a': 'name'}
        second_dict = {'b': 'age'}
        actual_dict = merge_config(first_dict, second_dict)
        self.assertEqual({'a': 'name', 'b': 'age'}, actual_dict)

    def test_three_dicts(self):
        first_dict = {'a': 'name'}
        second_dict = {'b': 'age'}
        third_dict = {'c': 'hair'}
        actual_dict = merge_config(first_dict, second_dict, third_dict)
        self.assertEqual({'a': 'name', 'b': 'age', 'c': 'hair'}, actual_dict)

    def test_order_matters(self):
        first_dict = {'a': 'name', 'b': 42}
        sec_dict = {'a': 'age'}
        third_dict = {'b': 15}
        actual_dict = merge_config(first_dict, sec_dict, third_dict)
        self.assertEqual({'a': 'age', 'b': 15}, actual_dict)

    def test_merge_deep(self):
        first_dict = {'a': dict(b=42, c=15), 'o': 105}
        second_dict = {'a': dict(c=25)}
        actual_dict = merge_config(first_dict, second_dict)
        self.assertEqual({'a': dict(b=42, c=25), 'o': 105}, actual_dict)

    def test_merge_dict_value(self):
        first_dict = {'o': 105}
        second_dict = {'a': dict(c=25)}
        actual_dict = merge_config(first_dict, second_dict)
        self.assertEqual({'a': dict(c=25), 'o': 105}, actual_dict)


# test/util/test_dask.py

import unittest

import numpy as np

from xcube.util.dask import _NestedList
from xcube.util.dask import compute_array_from_func
from xcube.util.dask import get_chunk_sizes
from xcube.util.dask import get_chunk_slice_tuples


class DaskTest(unittest.TestCase):
    def test_from_func_8x10_3x4(self):
        self._assert_from_func(shape=(8, 10), chunks=(3, 4))

    def test_from_func_80x100_3x4(self):
        self._assert_from_func(shape=(80, 100), chunks=(3, 4))

    def test_from_func_80x100_7x10(self):
        self._assert_from_func(shape=(80, 100), chunks=(7, 10))

    def test_from_func_80x100_40x25(self):
        self._assert_from_func(shape=(80, 100), chunks=(40, 25))

    def _assert_from_func(self, shape, chunks):
        def my_func(array_shape, array_dtype, block_shape, block_slices):
            bsy, bsx = block_slices
            bh, bw = block_shape
            w = array_shape[-1]
            i0 = bsx[0]
            j0 = bsy[0]
            a = np.ndarray((bh, bw), dtype=array_dtype)
            for j in range(bh):
                for i in range(bw):
                    a[j, i] = 0.1 * (i0 + i + (j0 + j) * w)
            return a

        a = compute_array_from_func(my_func,
                                    shape,
                                    chunks,
                                    np.float64,
                                    ctx_arg_names=['shape', 'dtype', 'block_shape', 'block_slices'])

        self.assertIsNotNone(a)
        self.assertEqual(shape, a.shape)
        self.assertEqual(tuple(get_chunk_sizes(shape, chunks)), a.chunks)
        self.assertEqual(np.float64, a.dtype)

        h, w = shape
        n = w * h

        # Compute result
        actual = np.array(a)
        expected = (0.1 * np.linspace(0, n - 1, n, dtype=np.float64)).reshape(shape)
        np.testing.assert_almost_equal(actual, expected)

    def test_get_chunk_sizes(self):
        chunks = get_chunk_sizes((100, 100, 40), (30, 25, 50))
        self.assertEqual([(30, 30, 30, 10),
                          (25, 25, 25, 25),
                          (40,)],
                         list(chunks))

        chunks = get_chunk_sizes((100, 100, 40), (100, 100, 40))
        self.assertEqual([(100,),
                          (100,),
                          (40,)],
                         list(chunks))

    def test_get_chunk_slices_tuples(self):
        chunk_slices_tuples = get_chunk_slice_tuples([(30, 30, 30, 10),
                                                      (25, 25, 25, 25),
                                                      (40,)])
        self.assertEqual([(slice(0, 30),
                           slice(30, 60),
                           slice(60, 90),
                           slice(90, 100)),
                          (slice(0, 25),
                           slice(25, 50),
                           slice(50, 75),
                           slice(75, 100)),
                          (slice(0, 40),)],
                         list(chunk_slices_tuples))


class NestedListTest(unittest.TestCase):
    def test_len(self):
        nl = _NestedList((4,))
        self.assertEqual(4, len(nl))

        nl = _NestedList((3, 4))
        self.assertEqual(3, len(nl))

        nl = _NestedList((2, 3, 4))
        self.assertEqual(2, len(nl))

    def test_shape(self):
        nl = _NestedList([4])
        self.assertEqual((4,), nl.shape)

        nl = _NestedList([3, 4])
        self.assertEqual((3, 4), nl.shape)

        nl = _NestedList([2, 3, 4])
        self.assertEqual((2, 3, 4), nl.shape)

    def test_data(self):
        nl = _NestedList((4,))
        self.assertEqual([None, None, None, None], nl.data)

        nl = _NestedList((3, 4))
        self.assertEqual([[None, None, None, None],
                          [None, None, None, None],
                          [None, None, None, None]], nl.data)

        nl = _NestedList((2, 3, 4))
        self.assertEqual([[[None, None, None, None],
                           [None, None, None, None],
                           [None, None, None, None]],
                          [[None, None, None, None],
                           [None, None, None, None],
                           [None, None, None, None]]], nl.data)

    def test_setget(self):
        nl = _NestedList((4,), fill_value=0)
        nl[1] = 2
        self.assertEqual([0, 2, 0, 0], nl.data)
        self.assertEqual(0, nl.data[0])
        self.assertEqual(2, nl.data[1])
        self.assertEqual(0, nl.data[2])
        nl[2:4] = [6, 7]
        self.assertEqual([0, 2, 6, 7], nl.data)

        nl = _NestedList((4, 3), fill_value=True)
        nl[2, 1] = False
        self.assertEqual([[True, True, True],
                          [True, True, True],
                          [True, False, True],
                          [True, True, True]], nl.data)
        nl[0, 0:2] = [False, False]
        self.assertEqual([[False, False, True],
                          [True, True, True],
                          [True, False, True],
                          [True, True, True]], nl.data)


# test/util/test_expression.py

import unittest
from collections import namedtuple

import numpy as np
import numpy.testing as npt
import xarray as xr

from xcube.util.expression import transpile_expr
from xcube.util.expression import compute_expr
from xcube.util.expression import compute_array_expr


class ComputeArrayExprTest(unittest.TestCase):
    # noinspection PyMethodMayBeStatic
    def test_valid_exprs(self):
        namespace = dict(a=np.array([0.1, 0.3, 0.1, 0.7, 0.4, 0.9]),
                         b=np.array([0.2, 0.1, 0.3, 0.2, 0.4, 0.8]),
                         np=np,
                         xr=xr)

        value = compute_array_expr('a + 1', namespace=namespace)
        npt.assert_array_almost_equal(value,
                                      np.array([1.1, 1.3, 1.1, 1.7, 1.4, 1.9]))

        value = compute_array_expr('a * b', namespace=namespace)
        npt.assert_array_almost_equal(value,
                                      np.array([0.02, 0.03, 0.03, 0.14, 0.16, 0.72]))

        value = compute_array_expr('max(a, b)', namespace=namespace)
        npt.assert_array_almost_equal(value,
                                      np.array([0.2, 0.3, 0.3, 0.7, 0.4, 0.9]))

        value = compute_array_expr('a > b', namespace=namespace)
        npt.assert_equal(value,
                         np.array([False, True, False, True, False, True]))

        value = compute_array_expr('a == b', namespace=namespace)
        npt.assert_equal(value,
                         np.array([False, False, False, False, True, False]))

        # This weirdo expression is a result of translating SNAP conditional expressions to Python.
        value = compute_array_expr('a > 0.35 if a else b', namespace=namespace)
        npt.assert_equal(value,
                         np.array([0.2, 0.1, 0.3, 0.7, 0.4, 0.9]))

        # We actually mean
        value = compute_array_expr('where(a > 0.35, a, b)', namespace=namespace)
        npt.assert_equal(value,
                         np.array([0.2, 0.1, 0.3, 0.7, 0.4, 0.9]))

    def test_invalid_exprs(self):
        with self.assertRaises(ValueError) as cm:
            compute_expr('20. -')
        self.assertEqual("failed computing expression '20. -': "
                         "unexpected EOF while parsing (<string>, line 1)",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            compute_expr('2 - a')
        self.assertEqual("failed computing expression '2 - a': "
                         "name 'a' is not defined",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            compute_expr('b < 1', result_name="mask of 'CHL'")
        self.assertEqual("failed computing mask of 'CHL' from "
                         "expression 'b < 1': name 'b' is not defined",
                         f'{cm.exception}')

    def test_complex_case(self):
        expr = ('(not quality_flags.invalid'
                ' and not pixel_classif_flags.IDEPIX_CLOUD'
                ' and not pixel_classif_flags.IDEPIX_CLOUD_BUFFER'
                ' and not pixel_classif_flags.IDEPIX_CLOUD_SHADOW'
                ' and not pixel_classif_flags.IDEPIX_SNOW_ICE'
                ' and not (c2rcc_flags.Rtosa_OOS and conc_chl > 1.0)'
                ' and not c2rcc_flags.Rtosa_OOR'
                ' and not c2rcc_flags.Rhow_OOR'
                ' and not (c2rcc_flags.Cloud_risk and immersed_cyanobacteria == 0)'
                ' and floating_vegetation == 0'
                ' and conc_chl > 0.01'
                ' and not (floating_cyanobacteria == 1 or chl_pitarch > 500))')

        quality_flags = namedtuple('quality_flags',
                                   ['invalid'])
        quality_flags.invalid = np.array([0])
        pixel_classif_flags = namedtuple('pixel_classif_flags',
                                         ['IDEPIX_CLOUD',
                                          'IDEPIX_CLOUD_BUFFER',
                                          'IDEPIX_CLOUD_SHADOW',
                                          'IDEPIX_SNOW_ICE'])
        pixel_classif_flags.IDEPIX_CLOUD = np.array([0])
        pixel_classif_flags.IDEPIX_CLOUD_BUFFER = np.array([0])
        pixel_classif_flags.IDEPIX_CLOUD_SHADOW = np.array([0])
        pixel_classif_flags.IDEPIX_SNOW_ICE = np.array([0])
        c2rcc_flags = namedtuple('c2rcc_flags',
                                 ['Rtosa_OOS',
                                  'Rtosa_OOR',
                                  'Rhow_OOR',
                                  'Cloud_risk'])
        c2rcc_flags.Rtosa_OOS = np.array([0])
        c2rcc_flags.Rtosa_OOR = np.array([0])
        c2rcc_flags.Rhow_OOR = np.array([0])
        c2rcc_flags.Cloud_risk = np.array([0])
        namespace = dict(
            np=np,
            quality_flags=quality_flags,
            pixel_classif_flags=pixel_classif_flags,
            c2rcc_flags=c2rcc_flags,
            immersed_cyanobacteria=np.array([0]),
            floating_cyanobacteria=np.array([0]),
            floating_vegetation=np.array([0]),
            conc_chl=np.array([0]),
            chl_pitarch=np.array([0]),
        )

        actual_value = compute_array_expr(expr, namespace=namespace)
        expected_value = 0
        npt.assert_array_almost_equal(actual_value, np.array([expected_value]))

        namespace['conc_chl'] = np.array([0.2])
        actual_value = compute_array_expr(expr, namespace=namespace)
        expected_value = 1
        npt.assert_array_almost_equal(actual_value, np.array([expected_value]))

        pixel_classif_flags.IDEPIX_CLOUD_SHADOW = np.array([0.2])
        actual_value = compute_array_expr(expr, namespace=namespace)
        expected_value = 0
        npt.assert_array_almost_equal(actual_value, np.array([expected_value]))


class ComputeExprTest(unittest.TestCase):
    def test_valid_exprs(self):
        namespace = dict(a=3)
        self.assertEqual(2, compute_expr('2'))
        self.assertEqual("u", compute_expr('"u"'))
        self.assertEqual(3, compute_expr('a', namespace=namespace))
        self.assertEqual(True, compute_expr('True'))
        self.assertEqual(True, compute_expr('a == 3', namespace=namespace))
        self.assertEqual(5, compute_expr('a + 2', namespace=namespace))

    def test_invalid_exprs(self):
        with self.assertRaises(ValueError) as cm:
            compute_expr('20. -')
        self.assertEqual("failed computing expression '20. -': "
                         "unexpected EOF while parsing (<string>, line 1)",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            compute_expr('2 - a')
        self.assertEqual("failed computing expression '2 - a': "
                         "name 'a' is not defined",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            compute_expr('b < 1', result_name="mask of 'CHL'")
        self.assertEqual("failed computing mask of 'CHL' from "
                         "expression 'b < 1': name 'b' is not defined",
                         f'{cm.exception}')


class TranspileExprTest(unittest.TestCase):
    def test_unary(self):
        self.assertEqual('--+-x',
                         transpile_expr('-(-+(-x))'))
        self.assertEqual('-(a - b - c)',
                         transpile_expr('-((a-b)-c)'))
        self.assertEqual('np.logical_not(x)',
                         transpile_expr('not x'))
        self.assertEqual('np.logical_not(-x)',
                         transpile_expr('not -x'))
        self.assertEqual('np.logical_not(np.logical_not(x))',
                         transpile_expr('not not x'))

    def test_binary(self):
        self.assertEqual('a - b - c - d',
                         transpile_expr('a-b-c-d'))
        self.assertEqual('a - b - c - d',
                         transpile_expr('(a-b)-c-d'))
        self.assertEqual('a - b - c - d',
                         transpile_expr('(a-b-c)-d'))
        self.assertEqual('a - (b - c) - d',
                         transpile_expr('a-(b-c)-d'))
        self.assertEqual('a - (b - c - d)',
                         transpile_expr('a-(b-c-d)'))
        self.assertEqual('a - b - (c - d)',
                         transpile_expr('a-b-(c-d)'))
        self.assertEqual('a - (b - (c - d))',
                         transpile_expr('a-(b-(c-d))'))
        self.assertEqual('a - (b - c) - d',
                         transpile_expr('(a-(b-c))-d'))

        self.assertEqual('a - ---b',
                         transpile_expr('a----b'))
        self.assertEqual('---a - b',
                         transpile_expr('---a-b'))

        self.assertEqual('a * b + c / d',
                         transpile_expr('a*b+c/d'))
        self.assertEqual('a + b * c - d',
                         transpile_expr('a+b*c-d'))
        self.assertEqual('(a + b) * (c - d)',
                         transpile_expr('(a+b)*(c-d)'))

        self.assertEqual('np.power(a, b)',
                         transpile_expr('a ** b'))

    def test_bool(self):
        self.assertEqual('np.logical_and(np.logical_and(a, b), c)',
                         transpile_expr('a and b and c'))
        self.assertEqual('np.logical_and(np.logical_and(a, b), c)',
                         transpile_expr('(a and b) and c'))
        self.assertEqual('np.logical_and(a, np.logical_and(b, c))',
                         transpile_expr('a and (b and c)'))
        self.assertEqual('np.logical_and(np.logical_or(a, b), '
                         'np.logical_not(np.logical_or(c, '
                         'np.logical_not(d))))',
                         transpile_expr('(a or b) and not (c or not d)'))

        self.assertEqual('np.logical_or(np.logical_and(a, b), '
                         'np.logical_and(c, d))',
                         transpile_expr('a and b or c and d'))
        self.assertEqual('np.logical_or(np.logical_or(a, '
                         'np.logical_and(b, c)), d)',
                         transpile_expr('a or b and c or d'))
        self.assertEqual('np.logical_and(np.logical_or(a, b), '
                         'np.logical_or(c, d))',
                         transpile_expr('(a or b) and (c or d)'))

    def test_compare(self):
        self.assertEqual('a < 2',
                         transpile_expr('a < 2'))
        self.assertEqual('a != 2',
                         transpile_expr('a != 2'))
        self.assertEqual('np.isnan(a)',
                         transpile_expr('a == NaN'))
        self.assertEqual('np.logical_not(np.isnan(a))',
                         transpile_expr('a != NaN'))
        self.assertEqual('np.isnan(a)',
                         transpile_expr('NaN == a'))
        self.assertEqual('np.logical_not(np.isnan(a))',
                         transpile_expr('NaN != a'))
        self.assertEqual('a is not Null',
                         transpile_expr('a is not Null'))
        self.assertEqual('a in data',
                         transpile_expr('a in data'))

        with self.assertRaises(ValueError) as cm:
            transpile_expr('a >= 2 == b')
        x8 = str(cm.exception)
        self.assertEqual('expression "a >= 2 == b" uses an n-ary '
                         'comparison, but only binary are supported', x8)

        with self.assertRaises(ValueError) as cm:
            transpile_expr('0 < x <= 1')
        x9 = str(cm.exception)
        self.assertEqual('expression "0 < x <= 1" uses an n-ary '
                         'comparison, but only binary are supported', x9)

    def test_attributes(self):
        self.assertEqual('a.b',
                         transpile_expr('a.b'))

    def test_functions(self):
        self.assertEqual('np.sin(x)',
                         transpile_expr('sin(x)'))
        self.assertEqual('np.fmin(x, y)',
                         transpile_expr('min(x, y)'))
        self.assertEqual('np.fmax(x, y)',
                         transpile_expr('max(x, y)'))
        self.assertEqual('np.isnan(x, y)',
                         transpile_expr('isnan(x, y)'))

    def test_conditional(self):
        # The following conditional expr looks wrong but this 
        # is how it looks like after translating
        # from SNAP expression 'a >= 0.0 ? a : NaN'
        self.assertEqual('xr.where(a >= 0.0, a, NaN)',
                         transpile_expr('a >= 0.0 if a else NaN'))
        self.assertEqual('xr.where(a >= 0.0, a, xr.where(b >= 0.0, b, NaN))',
                         transpile_expr('a >= 0.0 if a else b >= 0.0 if b else NaN'))

    def test_where(self):
        self.assertEqual('xr.where(a >= 0.0, a, NaN)',
                         transpile_expr('where(a >= 0.0, a, NaN)'))
        self.assertEqual('xr.where(a >= 0.0, a, NaN)',
                         transpile_expr('xr.where(a >= 0.0, a, NaN)'))
        self.assertEqual('np.where(a >= 0.0, a, NaN)',
                         transpile_expr('np.where(a >= 0.0, a, NaN)'))
        # xarray.DataArray.where() method:
        self.assertEqual('a.where(a.x >= 0.0)',
                         transpile_expr('a.where(a.x >= 0.0)'))

    def test_mixed(self):
        self.assertEqual('a + np.sin(x + 2.8)',
                         transpile_expr('a+sin(x + 2.8)'))
        self.assertEqual('a + np.fmax(1, np.sin(x + 2.8), np.power(x, 0.5))',
                         transpile_expr('a+max(1, sin(x+2.8), x**0.5)'))

        expr = ('(not quality_flags.invalid'
                ' and not pixel_classif_flags.IDEPIX_CLOUD'
                ' and not pixel_classif_flags.IDEPIX_CLOUD_BUFFER'
                ' and not pixel_classif_flags.IDEPIX_CLOUD_SHADOW'
                ' and not pixel_classif_flags.IDEPIX_SNOW_ICE'
                ' and not (c2rcc_flags.Rtosa_OOS and conc_chl > 1.0)'
                ' and not c2rcc_flags.Rtosa_OOR'
                ' and not c2rcc_flags.Rhow_OOR'
                ' and not (c2rcc_flags.Cloud_risk and immersed_cyanobacteria == 0)'
                ' and floating_vegetation == 0'
                ' and conc_chl > 0.01'
                ' and not (floating_cyanobacteria == 1 or chl_pitarch > 500))')

        self.assertEqual('np.logical_and(np.logical_and('
                         'np.logical_and(np.logical_and('
                         'np.logical_and(np.logical_and('
                         'np.logical_and(np.logical_and('
                         'np.logical_and(np.logical_and('
                         'np.logical_and('
                         'np.logical_not(quality_flags.invalid), '
                         'np.logical_not(pixel_classif_flags.IDEPIX_CLOUD)), '
                         'np.logical_not(pixel_classif_flags.IDEPIX_CLOUD_BUFFER)), '
                         'np.logical_not(pixel_classif_flags.IDEPIX_CLOUD_SHADOW)), '
                         'np.logical_not(pixel_classif_flags.IDEPIX_SNOW_ICE)), '
                         'np.logical_not('
                         'np.logical_and(c2rcc_flags.Rtosa_OOS, conc_chl > 1.0))), '
                         'np.logical_not(c2rcc_flags.Rtosa_OOR)), '
                         'np.logical_not(c2rcc_flags.Rhow_OOR)), '
                         'np.logical_not('
                         'np.logical_and(c2rcc_flags.Cloud_risk, immersed_cyanobacteria == 0))),'
                         ' floating_vegetation == 0), conc_chl > 0.01), '
                         'np.logical_not('
                         'np.logical_or(floating_cyanobacteria == 1, chl_pitarch > 500)))',
                         transpile_expr(expr))


# test/util/test_extension.py

import unittest
from typing import Any

from xcube.util.extension import ExtensionRegistry, Extension, get_extension_registry
from xcube.util.extension import import_component


class A:
    def __init__(self, **kwargs):
        self.kwargs = kwargs


class B(A):
    pass


class ExtensionRegistryTest(unittest.TestCase):
    def test_get_extension_reg(self):
        self.assertIsInstance(get_extension_registry(), ExtensionRegistry)

    def test_get_extension_illegal(self):
        ext_reg = ExtensionRegistry()
        self.assertEqual(False, ext_reg.has_extension('A', 'test'))
        self.assertEqual(None, ext_reg.get_extension('A', 'test'))

    def test_get_component_illegal(self):
        ext_reg = ExtensionRegistry()
        with self.assertRaises(ValueError) as cm:
            ext_reg.get_component('A', 'test')
        self.assertEqual("extension 'test' not found for extension point 'A'", f'{cm.exception}')

    def test_protocol(self):
        ext_reg = ExtensionRegistry()

        a_obj = A()

        def b_loader(ext):
            return B(name=ext.name)

        a_ext = ext_reg.add_extension(component=a_obj, point='A', name='test')
        b_ext = ext_reg.add_extension(loader=b_loader, point='B', name='test')

        self.assertEqual(True, ext_reg.has_extension('A', 'test'))
        self.assertIsInstance(a_ext, Extension)
        self.assertEqual('test', a_ext.name)
        self.assertEqual('A', a_ext.point)
        self.assertEqual(False, a_ext.is_lazy)
        self.assertIs(a_obj, a_ext.component)
        self.assertIs(a_obj, a_ext.component)
        self.assertIs(a_obj, ext_reg.get_component('A', 'test'))
        self.assertIs(a_ext, ext_reg.get_extension('A', 'test'))

        self.assertEqual(True, ext_reg.has_extension('B', 'test'))
        self.assertIsInstance(b_ext, Extension)
        self.assertEqual('test', b_ext.name)
        self.assertEqual('B', b_ext.point)
        self.assertEqual(True, b_ext.is_lazy)
        b_obj = b_ext.component
        self.assertIs(b_obj, b_ext.component)
        self.assertIs(b_obj, ext_reg.get_component('B', 'test'))
        self.assertIs(b_ext, ext_reg.get_extension('B', 'test'))
        self.assertEqual({'name': 'test'}, b_obj.kwargs)

        self.assertEqual([a_ext], ext_reg.find_extensions('A'))
        self.assertEqual([b_ext], ext_reg.find_extensions('B'))
        self.assertEqual([], ext_reg.find_extensions('C'))

        self.assertEqual([a_obj], ext_reg.find_components('A'))
        self.assertEqual([b_obj], ext_reg.find_components('B'))
        self.assertEqual([], ext_reg.find_components('C'))

        ext_reg.remove_extension('A', 'test')
        self.assertEqual(False, ext_reg.has_extension('A', 'test'))

        ext_reg.remove_extension('B', 'test')
        self.assertEqual(False, ext_reg.has_extension('B', 'test'))

    def test_find(self):
        ext_reg = ExtensionRegistry()
        obj1 = A()
        obj2 = A()
        obj3 = A()
        obj4 = B()
        obj5 = B()
        obj6 = B()

        # noinspection PyUnusedLocal
        def load_obj3(ext: Extension):
            return obj3

        # noinspection PyUnusedLocal
        def load_obj4(ext: Extension):
            return obj4

        ext_reg.add_extension(component=obj1, point='A', name='a1', description='knorg')
        ext_reg.add_extension(component=obj2, point='A', name='a2', description='gnatz')
        ext_reg.add_extension(loader=load_obj3, point='A', name='a3', description='gnatz')
        ext_reg.add_extension(loader=load_obj4, point='B', name='b1', description='knorg')
        ext_reg.add_extension(component=obj5, point='B', name='b2', description='gnatz')
        ext_reg.add_extension(component=obj6, point='B', name='b3', description='knorg')

        def is_knorg(ext: Extension):
            return ext.metadata.get('description') == 'knorg'

        def is_gnatz(ext: Extension):
            return ext.metadata.get('description') == 'gnatz'

        result = ext_reg.find_extensions('A', predicate=is_knorg)
        self.assertEqual(1, len(result))
        result = ext_reg.find_components('A', predicate=is_knorg)
        self.assertEqual(1, len(result))
        result = ext_reg.find_extensions('B', predicate=is_knorg)
        self.assertEqual(2, len(result))
        result = ext_reg.find_extensions('C', predicate=is_knorg)
        self.assertEqual(0, len(result))
        result = ext_reg.find_components('C', predicate=is_knorg)
        self.assertEqual(0, len(result))
        result = ext_reg.find_extensions('A', predicate=is_gnatz)
        self.assertEqual(2, len(result))
        result = ext_reg.find_components('A', predicate=is_gnatz)
        self.assertEqual(2, len(result))
        result = ext_reg.find_extensions('B', predicate=is_gnatz)
        self.assertEqual(1, len(result))
        result = ext_reg.find_components('B', predicate=is_gnatz)
        self.assertEqual(1, len(result))
        result = ext_reg.find_extensions('C', predicate=is_gnatz)
        self.assertEqual(0, len(result))
        result = ext_reg.find_components('C', predicate=is_gnatz)
        self.assertEqual(0, len(result))


class TestComponent:
    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs


class ImportTest(unittest.TestCase):
    def test_import_component(self):
        loader = import_component('test.util.test_extension:TestComponent')
        self.assertTrue(callable(loader))
        extension = Extension('test_point', 'test_component', component='dummy')
        component = loader(extension)
        self.assertIs(TestComponent, component)

    def test_import_and_transform_component(self):

        def transform(imported_component: Any, loaded_extension_: Extension):
            return imported_component(-1, name=loaded_extension_.name)

        loader = import_component('test.util.test_extension:TestComponent',
                                  transform=transform)
        self.assertTrue(callable(loader))
        extension = Extension('test_point', 'test_component', component='dummy')
        component = loader(extension)
        self.assertIsInstance(component, TestComponent)
        self.assertEqual((-1,), component.args)
        self.assertEqual({'name': 'test_component'}, component.kwargs)

    def test_import_component_and_call(self):
        loader = import_component('test.util.test_extension:TestComponent', call=True, call_args=[42],
                                  call_kwargs={'help': '!'})
        self.assertTrue(callable(loader))
        extension = Extension('test', 'test', component='x')
        component = loader(extension)
        self.assertIsInstance(component, TestComponent)
        self.assertEqual((42,), component.args)
        self.assertEqual({'help': '!'}, component.kwargs)


# test/util/test_geojson.py

import unittest

from xcube.util.geojson import GeoJSON


class GeoJSONTest(unittest.TestCase):

    def test_is_point(self):
        self.assertTrue(GeoJSON.is_point(dict(type='Point', coordinates=[2.13, 42.2])))
        self.assertFalse(GeoJSON.is_point(dict(type='Feature', properties=None)))

    def test_is_geometry(self):
        self.assertTrue(GeoJSON.is_geometry(dict(type='Point', coordinates=[2.13, 42.2])))
        self.assertTrue(GeoJSON.is_geometry(dict(type='Point', coordinates=None)))
        self.assertFalse(GeoJSON.is_geometry(dict(type='Point')))

        self.assertTrue(GeoJSON.is_geometry(dict(type='GeometryCollection', geometries=None)))
        self.assertTrue(GeoJSON.is_geometry(dict(type='GeometryCollection', geometries=[])))
        self.assertFalse(GeoJSON.is_geometry(dict(type='GeometryCollection')))

        self.assertFalse(GeoJSON.is_geometry(dict(type='Feature', properties=None)))

    def test_is_feature(self):
        self.assertTrue(GeoJSON.is_feature(dict(type='Feature',
                                                geometry=dict(type='Point',
                                                              coordinates=[2.13, 42.2]))))
        self.assertFalse(GeoJSON.is_feature(dict(type='Point', coordinates=[2.13, 42.2])))

    def test_is_feature_collection(self):
        self.assertTrue(GeoJSON.is_feature_collection(dict(type='FeatureCollection',
                                                           features=[dict(type='Feature',
                                                                          geometry=dict(type='Point',
                                                                                        coordinates=[
                                                                                            2.13,
                                                                                            42.2]))])))
        self.assertFalse(GeoJSON.is_feature_collection(dict(type='Point', coordinates=[2.13, 42.2])))

    def test_get_type_name(self):
        self.assertEqual('Feature',
                         GeoJSON.get_type_name(dict(type='Feature')))
        self.assertEqual(None,
                         GeoJSON.get_type_name(dict(pype='Feature')))
        self.assertEqual(None,
                         GeoJSON.get_type_name(dict()))
        self.assertEqual(None,
                         GeoJSON.get_type_name(17))

    def test_get_feature_geometry(self):
        self.assertEqual(dict(type='Point', coordinates=[2.13, 42.2]),
                         GeoJSON.get_feature_geometry(dict(type='Feature',
                                                           geometry=dict(type='Point', coordinates=[2.13, 42.2]))))
        self.assertEqual(None,
                         GeoJSON.get_feature_geometry(dict(type='Pleature',
                                                           geometry=dict(type='Point', coordinates=[2.13, 42.2]))))
        self.assertEqual(None,
                         GeoJSON.get_feature_geometry(dict(type='Feature',
                                                           geometry=dict(type='Point'))))
        self.assertEqual(None,
                         GeoJSON.get_feature_geometry(dict(type='Feature',
                                                           geometry=17)))


# test/util/test_ipython.py

import unittest

from xcube.util.ipython import register_json_formatter


class IPythonTest(unittest.TestCase):

    def test_it(self):
        class _FormatterTest1:
            def to_dict(self):
                return dict()

        # Should work
        register_json_formatter(_FormatterTest1)

        class _FormatterTest2:
            def to_dictomat(self):
                return dict()

        # Should not work
        with self.assertRaises(ValueError) as cm:
            register_json_formatter(_FormatterTest2)
        self.assertTrue(f'{cm.exception}'.endswith("FormatterTest2'> must define a to_dict() method"),
                        msg=f'{cm.exception}')


# test/util/test_jsonschema.py

import unittest
from collections import namedtuple
from typing import Dict, Any

from xcube.util.jsonschema import JsonArraySchema
from xcube.util.jsonschema import JsonBooleanSchema
from xcube.util.jsonschema import JsonComplexSchema
from xcube.util.jsonschema import JsonDateSchema
from xcube.util.jsonschema import JsonDatetimeSchema
from xcube.util.jsonschema import JsonIntegerSchema
from xcube.util.jsonschema import JsonNullSchema
from xcube.util.jsonschema import JsonNumberSchema
from xcube.util.jsonschema import JsonObject
from xcube.util.jsonschema import JsonObjectSchema
from xcube.util.jsonschema import JsonSimpleSchema
from xcube.util.jsonschema import JsonStringSchema


class JsonComplexSchemaTest(unittest.TestCase):

    def test_base_props_validated(self):
        with self.assertRaises(ValueError) as cm:
            JsonComplexSchema()
        self.assertEqual('exactly one of one_of, any_of, all_of must be given',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            JsonComplexSchema(one_of=[JsonStringSchema(), JsonIntegerSchema()],
                              all_of=[JsonStringSchema(), JsonIntegerSchema()])
        self.assertEqual('exactly one of one_of, any_of, all_of must be given',
                         f'{cm.exception}')

    def test_to_dict(self):
        self.assertEqual(
            {'oneOf': [{'multipleOf': 5, 'type': 'integer'},
                       {'multipleOf': 3, 'type': 'integer'}]},
            JsonComplexSchema(one_of=[JsonIntegerSchema(multiple_of=5),
                                      JsonIntegerSchema(multiple_of=3)]).to_dict())
        self.assertEqual(
            {'anyOf': [{'multipleOf': 5, 'type': 'integer'},
                       {'multipleOf': 3, 'type': 'integer'}]},
            JsonComplexSchema(any_of=[JsonIntegerSchema(multiple_of=5),
                                      JsonIntegerSchema(multiple_of=3)]).to_dict())
        self.assertEqual(
            {'allOf': [{'multipleOf': 5, 'type': 'integer'},
                       {'multipleOf': 3, 'type': 'integer'}]},
            JsonComplexSchema(all_of=[JsonIntegerSchema(multiple_of=5),
                                      JsonIntegerSchema(multiple_of=3)]).to_dict())


class JsonSimpleSchemaTest(unittest.TestCase):

    def test_base_props_validated(self):
        with self.assertRaises(ValueError) as cm:
            JsonSimpleSchema('int')
        self.assertEqual('type must be one of "array", "boolean", "integer", "null", "number", "object", "string"',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            JsonSimpleSchema('integer', factory='int')
        self.assertEqual('factory must be callable',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            JsonSimpleSchema('integer', serializer='int')
        self.assertEqual('serializer must be callable',
                         f'{cm.exception}')

    def test_to_dict(self):
        self.assertEqual(
            {
                'type': ['integer', 'null'],
                'title': 'Number of Gnarzes',
                'description': 'Not really required',
                'enum': [10, 100, 1000],
                'default': 10,
                'const': 100,
            },
            JsonSimpleSchema('integer',
                             title='Number of Gnarzes',
                             description='Not really required',
                             enum=[10, 100, 1000],
                             const=100,
                             default=10,
                             nullable=True).to_dict())


class JsonNullSchemaTest(unittest.TestCase):

    def test_to_json_null(self):
        self.assertEqual(None, JsonNullSchema().to_instance(None))

    def test_from_json_null(self):
        self.assertEqual(None, JsonNullSchema().from_instance(None))


class JsonBooleanSchemaTest(unittest.TestCase):

    def test_to_json_boolean(self):
        self.assertEqual(True, JsonBooleanSchema().to_instance(True))

    def test_from_json_boolean(self):
        self.assertEqual(True, JsonBooleanSchema().from_instance(True))


class JsonIntegerSchemaTest(unittest.TestCase):

    def test_to_instance(self):
        self.assertEqual(45, JsonIntegerSchema(minimum=0,
                                               maximum=100,
                                               multiple_of=5).to_instance(45))

    def test_from_instance(self):
        self.assertEqual(45, JsonIntegerSchema(minimum=0,
                                               maximum=100,
                                               multiple_of=5).from_instance(45))


class JsonNumberSchemaTest(unittest.TestCase):
    def test_to_instance(self):
        self.assertEqual(0.6, JsonNumberSchema(exclusive_minimum=0,
                                               exclusive_maximum=1).to_instance(0.6))

    def test_from_instance(self):
        self.assertEqual(6, JsonNumberSchema(minimum=0,
                                             maximum=10,
                                             multiple_of=2).from_instance(6))

    def test_check_type(self):
        with self.assertRaises(ValueError) as cm:
            JsonNumberSchema(type='float')
        self.assertEqual('Type must be one of "integer", "number"', f'{cm.exception}')

    def test_to_dict(self):
        self.assertEqual({'type': 'number'}, JsonNumberSchema().to_dict())
        self.assertEqual({'type': 'number'}, JsonNumberSchema(nullable=False).to_dict())
        self.assertEqual({'type': ['number', 'null']}, JsonNumberSchema(nullable=True).to_dict())
        self.assertEqual({'type': 'number',
                          'exclusiveMinimum': 0,
                          'maximum': 100,
                          'multipleOf': 10},
                         JsonNumberSchema(exclusive_minimum=0,
                                          maximum=100,
                                          multiple_of=10).to_dict())
        self.assertEqual({'type': 'integer',
                          'minimum': 100,
                          'exclusiveMaximum': 200,
                          'multipleOf': 20},
                         JsonIntegerSchema(minimum=100,
                                           exclusive_maximum=200,
                                           multiple_of=20).to_dict())


class JsonStringSchemaTest(unittest.TestCase):
    def test_to_instance(self):
        self.assertEqual('pieps', JsonStringSchema().to_instance('pieps'))
        self.assertEqual('pieps', JsonStringSchema(min_length=0,
                                                   max_length=10).to_instance('pieps'))
        self.assertEqual('pieps', JsonStringSchema(pattern='.*').to_instance('pieps'))
        self.assertEqual('2020-01-03T03:30:01.99+03:30',
                         JsonStringSchema(format='date-time').
                         to_instance('2020-01-03T03:30:01.99+03:30'))

    def test_from_instance(self):
        self.assertEqual('pieps', JsonStringSchema().from_instance('pieps'))

    def test_to_dict(self):
        self.assertEqual({'type': 'string'}, JsonStringSchema().to_dict())
        self.assertEqual({'type': 'string'}, JsonStringSchema(nullable=False).to_dict())
        self.assertEqual({'type': ['string', 'null']}, JsonStringSchema(nullable=True).to_dict())
        self.assertEqual({'type': 'string', 'format': 'uri'}, JsonStringSchema(format='uri').to_dict())


class JsonDateSchemaTest(unittest.TestCase):

    def test_to_instance(self):
        self.assertEqual('2020-06-03',
                         JsonDateSchema(min_date='2020-02-01',
                                        max_date='2020-07-05').to_instance('2020-06-03'))

    def test_to_dict(self):
        self.assertEqual({'type': 'string',
                          'format': 'date',
                          'minDate': '2020-02-01',
                          'maxDate': '2020-07-05'
                          },
                         JsonDateSchema(min_date='2020-02-01',
                                        max_date='2020-07-05').to_dict())

        self.assertEqual({'type': ['string', 'null'],
                          'format': 'date',
                          'minDate': '2020-02-01',
                          'maxDate': '2020-07-05'
                          },
                         JsonDateSchema(min_date='2020-02-01',
                                        max_date='2020-07-05',
                                        nullable=True).to_dict())

    def test_store_date_limits(self):
        minimum = '1981-05-06'
        maximum = '1982-09-15'
        schema = JsonDateSchema(min_date=minimum,
                                max_date=maximum)
        self.assertEqual(minimum, schema.min_date)
        self.assertEqual(maximum, schema.max_date)

    def test_min_max_validity_checks(self):
        with self.assertRaises(ValueError):
            JsonDateSchema(min_date='2002-02-02T10:20:14')
        with self.assertRaises(ValueError):
            JsonDateSchema(max_date='pippo')

    def test_new_range(self):
        self.assertEqual(
            {'type': 'array',
             'items': [{'type': 'string',
                        'format': 'date',
                        },
                       {'type': 'string',
                        'format': 'date',
                        }
                       ],
             },
            JsonDateSchema.new_range().to_dict())

        self.assertEqual(
            {'type': ['array', 'null'],
             'items': [{'type': ['string', 'null'],
                        'format': 'date',
                        'minDate': '2020-02-01',
                        'maxDate': '2020-07-05',
                        },
                       {'type': ['string', 'null'],
                        'format': 'date',
                        'minDate': '2020-02-01',
                        'maxDate': '2020-07-05',
                        }
                       ],
             },
            JsonDateSchema.new_range(min_date='2020-02-01',
                                     max_date='2020-07-05',
                                     nullable=True).to_dict())


class JsonDatetimeSchemaTest(unittest.TestCase):

    def test_to_instance(self):
        self.assertEqual('2020-06-12T12:30:19Z',
                         JsonDatetimeSchema(min_datetime='2020-02-01T00:00:00Z',
                                            max_datetime='2020-07-05T00:00:00Z').to_instance(
                             '2020-06-12T12:30:19Z'))

    def test_to_dict(self):
        self.assertEqual({'type': 'string',
                          'format': 'date-time',
                          'minDatetime': '2020-02-01T00:00:00Z',
                          'maxDatetime': '2020-07-05T00:00:00Z'},
                         JsonDatetimeSchema(min_datetime='2020-02-01T00:00:00Z',
                                            max_datetime='2020-07-05T00:00:00Z').to_dict())

    def test_store_date_limits(self):
        minimum = '1981-05-06T00:00:00+00:00'
        maximum = '1982-09-15T00:00:00+00:00'
        schema = JsonDatetimeSchema(min_datetime=minimum,
                                    max_datetime=maximum)
        self.assertEqual(minimum, schema.min_datetime)
        self.assertEqual(maximum, schema.max_datetime)

    def test_min_max_validity_checks(self):
        with self.assertRaises(ValueError):
            JsonDatetimeSchema(  # missing timezone specifier
                min_datetime='1980-02-03T12:34:56',
                max_datetime='1982-02-03T23:34:56+05:00')
        with self.assertRaises(ValueError):
            JsonDatetimeSchema(min_datetime='1980-02-03T12:34:56-08:00',
                               # invalid date
                               max_datetime='1985-01-32')

    def test_new_range(self):
        self.assertEqual(
            {'type': 'array',
             'items': [{'type': 'string',
                        'format': 'date-time',
                        },
                       {'type': 'string',
                        'format': 'date-time',
                        }
                       ],
             },
            JsonDatetimeSchema.new_range().to_dict())

        self.assertEqual(
            {'type': ['array', 'null'],
             'items': [{'type': ['string', 'null'],
                        'format': 'date-time',
                        'minDatetime': '2002-01-01T00:00:00Z',
                        'maxDatetime': '2020-01-01T00:00:00Z',
                        },
                       {'type': ['string', 'null'],
                        'format': 'date-time',
                        'minDatetime': '2002-01-01T00:00:00Z',
                        'maxDatetime': '2020-01-01T00:00:00Z',
                        }
                       ],
             },
            JsonDatetimeSchema.new_range(min_datetime='2002-01-01T00:00:00Z',
                                         max_datetime='2020-01-01T00:00:00Z',
                                         nullable=True).to_dict())


class JsonArraySchemaTest(unittest.TestCase):

    def test_to_instance(self):
        self.assertEqual([False, 2, 'U'],
                         JsonArraySchema().to_instance([False, 2, 'U']))

    def test_to_instance_tuple(self):
        self.assertEqual([False, 2, 'U'],
                         JsonArraySchema(items=[JsonBooleanSchema(),
                                                JsonIntegerSchema(),
                                                JsonStringSchema()]).to_instance([False, 2, 'U']))

    def test_from_instance(self):
        self.assertEqual([False, 2, 'U'],
                         JsonArraySchema().from_instance([False, 2, 'U']))

    def test_from_instance_tuple(self):
        self.assertEqual([False, 2, 'U'],
                         JsonArraySchema(items=[JsonBooleanSchema(),
                                                JsonIntegerSchema(),
                                                JsonStringSchema()]).from_instance([False, 2, 'U']))

    def test_tuple_validates_as_array(self):
        self.assertTupleEqual((1, 2, 3),
                              JsonArraySchema().to_instance((1, 2, 3)))

    def test_from_instance_array_object(self):
        value = [{'name': 'Bibo', 'age': 15},
                 {'name': 'Ernie', 'age': 12}]

        person_schema = JsonObjectSchema(
            properties=dict(name=JsonStringSchema(), age=JsonIntegerSchema(),
                            deleted=JsonBooleanSchema(default=False)))
        schema = JsonArraySchema(items=person_schema)
        self.assertEqual([{'name': 'Bibo', 'age': 15, 'deleted': False},
                          {'name': 'Ernie', 'age': 12, 'deleted': False}],
                         schema.from_instance(value))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])
        person_schema.factory = Person
        self.assertEqual([Person(name='Bibo', age=15, deleted=False),
                          Person(name='Ernie', age=12, deleted=False)],
                         schema.from_instance(value))

        Assignment = namedtuple('Assignment', ['persons'])

        def factory(persons):
            return Assignment(persons=persons)

        schema.factory = factory
        self.assertEqual(Assignment(persons=[Person(name='Bibo', age=15, deleted=False),
                                             Person(name='Ernie', age=12, deleted=False)]),
                         schema.from_instance(value))


class JsonObjectSchemaTest(unittest.TestCase):

    def test_from_json_object(self):
        value = {'name': 'Bibo', 'age': 12, 'deleted': True}

        person_schema = JsonObjectSchema(properties=dict(name=JsonStringSchema(),
                                                         age=JsonIntegerSchema(),
                                                         deleted=JsonBooleanSchema(default=False)))
        self.assertEqual(value,
                         person_schema.from_instance(value))

        self.assertEqual({'name': 'Bibo', 'age': 12, 'deleted': False},
                         person_schema.from_instance({'name': 'Bibo', 'age': 12}))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])
        person_schema.factory = Person
        self.assertEqual(Person(name='Bibo', age=12, deleted=True),
                         person_schema.from_instance(value))

    def test_to_json_object(self):
        person_schema = JsonObjectSchema(properties=dict(name=JsonStringSchema(),
                                                         age=JsonIntegerSchema(),
                                                         deleted=JsonBooleanSchema(default=False)))

        value = {'name': 'Bibo', 'age': 12, 'deleted': True}

        self.assertEqual(value,
                         person_schema.to_instance(value))

        # ok, because person_schema does not explicitly say additional_properties=False
        value_extra = {'name': 'Bibo', 'age': 12, 'deleted': True, 'comment': 'Hello!'}
        self.assertEqual(value_extra,
                         person_schema.to_instance(value_extra))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])

        def serialize(person: Person) -> Dict[str, Any]:
            return person._asdict()

        person_schema.serializer = serialize

        person = Person(**value)
        self.assertEqual(value,
                         person_schema.to_instance(person))

    def test_from_json_object_object(self):
        person_schema = JsonObjectSchema(properties=dict(name=JsonStringSchema(),
                                                         age=JsonIntegerSchema(),
                                                         deleted=JsonBooleanSchema(default=False)))
        schema = JsonObjectSchema(properties=dict(person=person_schema))

        value = {'person': {'name': 'Bibo', 'age': 15}}

        self.assertEqual({'person': {'name': 'Bibo', 'age': 15, 'deleted': False}},
                         schema.from_instance(value))

        Assignment = namedtuple('Assignment', ['person'])
        schema.factory = Assignment
        self.assertEqual(Assignment(person={'name': 'Bibo', 'age': 15, 'deleted': False}),
                         schema.from_instance(value))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])
        person_schema.factory = Person
        self.assertEqual(Assignment(person=Person(name='Bibo', age=15, deleted=False)),
                         schema.from_instance(value))

    def test_from_json_object_array_object(self):
        person_schema = JsonObjectSchema(properties=dict(name=JsonStringSchema(),
                                                         age=JsonIntegerSchema(),
                                                         deleted=JsonBooleanSchema(default=False)))

        schema = JsonObjectSchema(properties=dict(persons=JsonArraySchema(items=person_schema)))

        value = {'persons': [{'name': 'Bibo', 'age': 15},
                             {'name': 'Ernie', 'age': 12}]}

        self.assertEqual({'persons': [{'name': 'Bibo', 'age': 15, 'deleted': False},
                                      {'name': 'Ernie', 'age': 12, 'deleted': False}]},
                         schema.from_instance(value))

        Assignment = namedtuple('Assignment', ['persons'])
        schema.factory = Assignment
        self.assertEqual(Assignment(persons=[{'name': 'Bibo', 'age': 15, 'deleted': False},
                                             {'name': 'Ernie', 'age': 12, 'deleted': False}]),
                         schema.from_instance(value))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])
        person_schema.factory = Person
        self.assertEqual(Assignment(persons=[Person(name='Bibo', age=15, deleted=False),
                                             Person(name='Ernie', age=12, deleted=False)]),
                         schema.from_instance(value))

    def test_from_json_object_additional_properties_is_schema(self):
        Person = namedtuple('Person', ['name', 'age', 'deleted'])

        person_schema = JsonObjectSchema(
            properties=dict(
                name=JsonStringSchema(),
                age=JsonIntegerSchema(),
                deleted=JsonBooleanSchema(default=False)
            ),
            factory=Person,
        )

        schema = JsonObjectSchema(
            additional_properties=person_schema,
        )

        value = {
            'p1': {'name': 'Bibo', 'age': 15, 'deleted': True},
            'p2': {'name': 'Ernie', 'age': 12, 'deleted': False},
        }

        self.assertEqual(
            {
                'p1': Person(name='Bibo', age=15, deleted=True),
                'p2': Person(name='Ernie', age=12, deleted=False),
            },
            schema.from_instance(value))

    def test_process_kwargs_subset(self):
        schema = JsonObjectSchema(
            properties=dict(
                client_id=JsonStringSchema(default='bibo'),
                client_secret=JsonStringSchema(default='2w908435t'),
                geom=JsonStringSchema(),
                crs=JsonStringSchema(const='WGS84'),
                spatial_res=JsonNumberSchema(),
                time_range=JsonStringSchema(),
                time_period=JsonStringSchema(default='8D'),
                max_cache_size=JsonIntegerSchema(),
            ),
            required=['client_id', 'client_secret', 'geom', 'crs', 'spatial_res', 'time_range'],
        )

        kwargs = dict(client_secret='094529g',
                      geom='POINT (12.2, 53.9)',
                      spatial_res=0.5,
                      time_range='2010,2014',
                      max_cache_size=2 ** 32)

        cred_kwargs, kwargs = schema.process_kwargs_subset(kwargs, ['client_id', 'client_secret'])
        self.assertEqual(dict(client_id='bibo', client_secret='094529g'),
                         cred_kwargs)
        self.assertEqual(dict(geom='POINT (12.2, 53.9)',
                              spatial_res=0.5,
                              time_range='2010,2014',
                              max_cache_size=2 ** 32),
                         kwargs)

        ds_kwargs, kwargs = schema.process_kwargs_subset(kwargs, ['geom', 'crs',
                                                                  'spatial_res', 'time_range', 'time_period'])
        self.assertEqual(dict(crs='WGS84',
                              geom='POINT (12.2, 53.9)',
                              spatial_res=0.5,
                              time_range='2010,2014'),
                         ds_kwargs)
        self.assertEqual(dict(max_cache_size=2 ** 32),
                         kwargs)

    def test_to_dict(self):
        schema = JsonObjectSchema(
            properties=dict(
                consolidated=JsonBooleanSchema()
            )
        )
        self.assertEqual(
            {
                'type': 'object',
                'properties': {
                    'consolidated': {
                        'type': 'boolean'
                    }
                },
            },
            schema.to_dict()
        )


class IHasAProperty(JsonObject):
    def __init__(self, name, age):
        # A private state property
        self._name = name
        # A state property
        self.age = age

    @property
    def id(self):
        # A derived property
        return f'{self.name}_{self.age}'

    @property
    def name(self):
        # A state property
        return self._name

    @classmethod
    def get_schema(cls):
        return JsonObjectSchema(
            properties=dict(
                name=JsonStringSchema(),
                age=JsonIntegerSchema()
            ),
            required=['name', 'age'],
            factory=cls
        )


class IHasAProperty2(IHasAProperty):
    @classmethod
    def get_schema(cls):
        schema = super().get_schema()
        schema.additional_properties = False
        return schema


class JsonObjectWithPropertyTest(unittest.TestCase):
    def test_with_additional_properties(self):
        obj_dict = {'name': 'bibo', 'age': 2021 - 1969}
        obj = IHasAProperty.from_dict(obj_dict)
        actual_dict = obj.to_dict()
        self.assertEqual({'age': 52, 'id': 'bibo_52', 'name': 'bibo'}, actual_dict)

    def test_without_additional_properties(self):
        obj_dict = {'name': 'bibo', 'age': 2021 - 1969}
        obj = IHasAProperty2.from_dict(obj_dict)
        actual_dict = obj.to_dict()
        self.assertEqual(obj_dict, actual_dict)


# test/util/test_ndarray_to_cmap_image.py

import copy
from unittest import TestCase

import numpy as np
from PIL import Image
from matplotlib import cm


# * http://stackoverflow.com/questions/14448763/is-there-a-convenient-way-to-apply-a-lookup-table-to-a-large-array-in-numpy
# * http://stackoverflow.com/questions/10965417/how-to-convert-numpy-array-to-pil-image-applying-matplotlib-colormap
# * http://stackoverflow.com/questions/10443295/combine-3-separate-numpy-arrays-to-an-rgb-image-in-python
# * http://stackoverflow.com/questions/2578752/how-can-i-plot-nan-values-as-a-special-color-with-imshow-in-matplotlib
#
# * http://matplotlib.org/api/cm_api.html#module-matplotlib.cm
# * http://matplotlib.org/users/colormaps.html
# * http://matplotlib.org/api/colors_api.html#
# * https://bids.github.io/colormap/

class PILImageTest(TestCase):
    def test_array_to_rgba(self):
        nan = np.nan
        array = np.array([
            [0.0, 0.1, 0.2, nan],
            [0.2, 0.3, nan, 0.5],
            [0.4, nan, 0.6, 0.7],
            [nan, 0.7, 0.8, 0.9],
        ])
        # print(array, array.dtype)

        min = 0.1
        max = 0.8

        array.clip(min, max, out=array)
        # print(array, array.dtype)

        array -= min
        array *= 1.0 / (max - min)
        # print(array, array.dtype)

        array = np.ma.masked_invalid(array, copy=False)
        # print(array, array.dtype)

        # import pprint
        # pprint.pprint(dir(cm.gist_earth))

        # We want to have transparency where the array is masked out.
        # This is done via colormap.set_bad(color, alpha).
        # However, in doing we would modify the global cm.gist_earth singleton so we create a copy here:
        #
        cmap_copy = copy.copy(cm.gist_earth)
        cmap_copy.set_bad('k', 0)
        array = cmap_copy(array, bytes=True)
        # print(array, array.dtype)
        self.assertEqual([[[0, 0, 0, 255],
                           [0, 0, 0, 255],
                           [23, 64, 121, 255],
                           [0, 0, 0, 0],
                           ],

                          [[23, 64, 121, 255],
                           [48, 128, 126, 255],
                           [0, 0, 0, 0],
                           [130, 168, 83, 255],
                           ],

                          [[65, 149, 82, 255],
                           [0, 0, 0, 0],
                           [184, 179, 95, 255],
                           [212, 176, 147, 255],
                           ],

                          [[0, 0, 0, 0],
                           [212, 176, 147, 255],
                           [253, 250, 250, 255],
                           [253, 250, 250, 255],
                           ],
                          ], array.tolist())

        im = Image.fromarray(array, mode='RGBA')

        # im.show('Raw Image')

        self.assertEqual((0, 0, 0, 255), im.getpixel((0, 0)))
        self.assertEqual((23, 64, 121, 255), im.getpixel((2, 0)))
        self.assertEqual((0, 0, 0, 0), im.getpixel((1, 2)))
        self.assertEqual((212, 176, 147, 255), im.getpixel((1, 3)))


# test/util/test_perf.py

import time
from unittest import TestCase

from xcube.util.perf import measure_time_cm


class MeasureTimeTest(TestCase):
    def test_enabled(self):
        measure_time = measure_time_cm(disabled=False)
        with measure_time("hello") as cm:
            time.sleep(0.06)
        self.assertTrue(hasattr(cm, "duration"))
        self.assertTrue(cm.duration > 0.05)
        self.assertIsNotNone(cm.logger)

    def test_disabled(self):
        measure_time = measure_time_cm(disabled=True)
        with measure_time("hello") as cm:
            time.sleep(0.05)
        self.assertTrue(hasattr(cm, "duration"))
        self.assertIsNone(cm.duration)
        self.assertIsNone(cm.logger)


# test/util/test_plugin.py

import unittest

from xcube.util.extension import ExtensionRegistry
from xcube.util.plugin import get_plugins, load_plugins, discover_plugin_modules


def init_plugin(ext_registry: ExtensionRegistry):
    """A test plugin that registers test extensions"""
    ext_registry.add_extension(component=object(), point='test.util.test_plugin', name='ext1')
    ext_registry.add_extension(component=object(), point='test.util.test_plugin', name='ext2')
    ext_registry.add_extension(component=object(), point='test.util.test_plugin', name='ext3')


def init_plugin_bad():
    raise RuntimeError()


class EntryPoint:
    def __init__(self, name, func):
        self.name = name
        self.func = func

    def load(self):
        return self.func


class EntryPointBad(EntryPoint):

    def load(self):
        raise RuntimeError()


class PluginTest(unittest.TestCase):

    def setUp(self):
        self.ext_registry = ExtensionRegistry()

    def test_get_xcube_default_plugins(self):
        plugins = get_plugins()
        self.assertIsNotNone(plugins)
        self.assertIn('xcube', plugins)

    def test_load_plugins_by_entry_points(self):
        plugins = load_plugins([EntryPoint('test', init_plugin)], ext_registry=self.ext_registry)
        self.assertEqual(dict(test=dict(name='test', doc='A test plugin that registers test extensions')), plugins)
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext1'))
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext2'))
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext3'))

    def test_load_plugins_by_module_discovery(self):
        entry_points = discover_plugin_modules(module_prefixes=['test'])
        plugins = load_plugins(entry_points, ext_registry=self.ext_registry)
        self.assertEqual(dict(test=dict(name='test', doc='A test plugin that registers test extensions')), plugins)
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext1'))
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext2'))
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext3'))

    def test_load_plugins_by_bad_entry_point(self):
        plugins = load_plugins([EntryPointBad('test', init_plugin)], ext_registry=self.ext_registry)
        self.assertEqual({}, plugins)
        self.assertEqual([], self.ext_registry.find_components('test.util.test_plugin'))

    def test_load_plugins_by_bad_init_plugin(self):
        plugins = load_plugins([EntryPoint('test', init_plugin_bad)], ext_registry=self.ext_registry)
        self.assertEqual({}, plugins)
        self.assertEqual([], self.ext_registry.find_components('test.util.test_plugin'))

    def test_load_plugins_init_plugin_not_callable(self):
        plugins = load_plugins([EntryPoint('test', "init_plugin_not_callable")], ext_registry=self.ext_registry)
        self.assertEqual({}, plugins)
        self.assertEqual([], self.ext_registry.find_components('test'))

    def test_load_plugins_by_failing_module_discovery(self):
        entry_points = discover_plugin_modules(module_prefixes=['random'])
        plugins = load_plugins(entry_points, ext_registry=self.ext_registry)
        self.assertEqual({}, plugins)
        self.assertEqual([], self.ext_registry.find_components('test.util.test_plugin'))


# test/util/test_progress.py

import unittest
from typing import Sequence

import dask
import dask.array.random

from xcube.util.progress import ProgressObserver
from xcube.util.progress import ProgressState
from xcube.util.progress import add_progress_observers
from xcube.util.progress import new_progress_observers
from xcube.util.progress import observe_dask_progress
from xcube.util.progress import observe_progress


class TestProgressObserver(ProgressObserver):
    def __init__(self, record_errors=False):
        self.record_errors = record_errors
        self.calls = []

    def on_begin(self, state_stack: Sequence[ProgressState]):
        self.calls.append(('begin', self._serialize_stack(state_stack)))

    def on_update(self, state_stack: Sequence[ProgressState]):
        self.calls.append(('update', self._serialize_stack(state_stack)))

    def on_end(self, state_stack: Sequence[ProgressState]):
        self.calls.append(('end', self._serialize_stack(state_stack)))

    def _serialize_stack(self, state_stack):
        if self.record_errors:
            return [(s.label, s.progress, s.finished, s.exc_info_text) for s in state_stack]
        else:
            return [(s.label, s.progress, s.finished) for s in state_stack]


class ObserveProgressTest(unittest.TestCase):

    def test_observe_progress(self):
        observer = TestProgressObserver()
        observer.activate()

        with observe_progress('computing', 4) as reporter:
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that takes 2 units
            reporter.worked(2)

        self.assertIsInstance(reporter.state, ProgressState)
        self.assertIsInstance(reporter.state.total_time, float)
        self.assertTrue(reporter.state.total_time >= 0.0)

        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False)]),
                ('update', [('computing', 0.25, False)]),
                ('update', [('computing', 0.5, False)]),
                ('update', [('computing', 1.0, False)]),
                ('end', [('computing', 1.0, True)])
            ],
            observer.calls)

    def test_nested_observe_progress(self):
        observer = TestProgressObserver()
        observer.deactivate()
        observer.activate()

        with observe_progress('computing', 4) as reporter:
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that will take 2 units
            reporter.will_work(2)
            with observe_progress('loading', 4) as nested_reporter:
                # do something that takes 3 units
                nested_reporter.worked(3)
                # do something that takes 1 unit
                nested_reporter.worked(1)

        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False)]),
                ('update', [('computing', 0.25, False)]),
                ('update', [('computing', 0.5, False)]),
                ('begin', [('computing', 0.5, False), ('loading', 0.0, False)]),
                ('update', [('computing', 0.875, False), ('loading', 0.75, False)]),
                ('update', [('computing', 1.0, False), ('loading', 1.0, False)]),
                ('end', [('computing', 1.0, False), ('loading', 1.0, True)]),
                ('end', [('computing', 1.0, True)])
            ],
            observer.calls)

    def test_nested_observe_progress_with_new_progress_observers(self):
        observer = TestProgressObserver()
        observer.activate()

        nested_observer = TestProgressObserver()

        with observe_progress('computing', 4) as progress_reporter:
            # do something that takes 1 unit
            progress_reporter.worked(1)
            # do something that takes 1 unit
            progress_reporter.worked(1)
            with new_progress_observers(nested_observer):
                with observe_progress('loading', 4) as progress_reporter_2:
                    # do something that takes 3 units
                    progress_reporter_2.worked(3)
                    # do something that takes 1 unit
                    progress_reporter_2.worked(1)

            # do something that takes 1 unit
            progress_reporter.worked(2)

        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False)]),
                ('update', [('computing', 0.25, False)]),
                ('update', [('computing', 0.5, False)]),
                ('update', [('computing', 1.0, False)]),
                ('end', [('computing', 1.0, True)])
            ],
            observer.calls)

        self.assertEqual(
            [
                ('begin', [('loading', 0.0, False)]),
                ('update', [('loading', 0.75, False)]),
                ('update', [('loading', 1.0, False)]),
                ('end', [('loading', 1.0, True)])
            ],
            nested_observer.calls)

    def test_nested_observe_progress_with_add_progress_observers(self):
        observer = TestProgressObserver()
        observer.activate()

        nested_observer = TestProgressObserver()

        with observe_progress('computing', 4) as reporter:
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that will take 2 units
            reporter.will_work(2)
            with add_progress_observers(nested_observer):
                with observe_progress('loading', 4) as nested_reported:
                    # do something that takes 3 units
                    nested_reported.worked(3)
                    # do something that takes 1 unit
                    nested_reported.worked(1)

        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False)]),
                ('update', [('computing', 0.25, False)]),
                ('update', [('computing', 0.5, False)]),
                ('begin', [('computing', 0.5, False), ('loading', 0.0, False)]),
                ('update', [('computing', 0.875, False), ('loading', 0.75, False)]),
                ('update', [('computing', 1.0, False), ('loading', 1.0, False)]),
                ('end', [('computing', 1.0, False), ('loading', 1.0, True)]),
                ('end', [('computing', 1.0, True)])
            ],
            observer.calls)

        self.assertEqual(
            [
                ('begin', [('computing', 0.5, False), ('loading', 0.0, False)]),
                ('update', [('computing', 0.875, False), ('loading', 0.75, False)]),
                ('update', [('computing', 1.0, False), ('loading', 1.0, False)]),
                ('end', [('computing', 1.0, False), ('loading', 1.0, True)])
            ],
            nested_observer.calls)

    def test_nested_observe_progress_with_exception(self):
        observer = TestProgressObserver(record_errors=True)
        observer.activate()

        try:
            with observe_progress('computing', 10) as reporter:
                # do something that takes 1 unit
                reporter.worked(1)
                # do something that takes 1 unit
                reporter.worked(1)
                # do something that will take 2 units
                reporter.will_work(8)
                with observe_progress('loading', 100) as nested_reported:
                    # do something that takes 3 units
                    nested_reported.worked(15)
                    # now - BANG!
                    raise ValueError('Failed to load')
        except ValueError:
            pass
        self.assertEqual(7, len(observer.calls))
        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False, None)]),
                ('update', [('computing', 0.1, False, None)]),
                ('update', [('computing', 0.2, False, None)]),
                ('begin', [('computing', 0.2, False, None), ('loading', 0.0, False, None)]),
                ('update', [('computing', 0.32, False, None), ('loading', 0.15, False, None)]),
            ],
            observer.calls[0: -2])

        self.assertEqual(2, len(observer.calls[-2]))
        event, states = observer.calls[-2]
        self.assertEqual('end', event)
        self.assertEqual(2, len(states))
        self.assertEqual(4, len(states[0]))
        self.assertEqual(4, len(states[1]))
        self.assertEqual(('computing', 0.32, False), states[0][0:-1])
        self.assertEqual(('loading', 0.15, True), states[1][0:-1])
        error = states[0][-1]
        self.assertIsNone(error)
        error = states[1][-1]
        self.assertIsInstance(error, tuple)
        exc_type, exc_value, exc_traceback = error
        self.assertEqual('ValueError', exc_type)
        self.assertEqual('Failed to load', exc_value)
        self.assertIsInstance(exc_traceback, list)

        self.assertEqual(2, len(observer.calls[-1]))
        event, states = observer.calls[-1]
        self.assertEqual('end', event)
        self.assertEqual(1, len(states))
        self.assertEqual(4, len(states[0]))
        self.assertEqual(('computing', 0.32, True), states[0][0:-1])
        error = states[0][-1]
        self.assertIsInstance(error, tuple)
        exc_type, exc_value, exc_traceback = error
        self.assertEqual('ValueError', exc_type)
        self.assertEqual('Failed to load', exc_value)
        self.assertIsInstance(exc_traceback, list)

    def test_dask_progress(self):
        observer = TestProgressObserver(record_errors=True)
        observer.activate()

        res = dask.array.random.normal(size=(100, 200), chunks=(25, 50))
        with observe_dask_progress('computing', 100):
            res.compute()

        self.assertEqual(4, len(res.chunks[0]))
        self.assertTrue(len(observer.calls) >= 3)
        self.assertEqual(('begin', [('computing', 0.0, False, None)]), observer.calls[0])
        self.assertEqual(('update', [('computing', 5 / 16, False, None)]), observer.calls[5])
        self.assertEqual(('end', [('computing', 15 / 16, True, None)]), observer.calls[-1])


class ProgressStateTest(unittest.TestCase):

    def test_progress_state_props(self):
        state = ProgressState('computing', 100, 3)
        self.assertEqual('computing', state.label)
        self.assertEqual(100, state.total_work)
        self.assertEqual(3, state.super_work)
        self.assertEqual(1, state.super_work_ahead)
        self.assertEqual(0, state.completed_work)
        self.assertEqual(0, state.progress)
        self.assertEqual(0.6, state.to_super_work(20))
        state.inc_work(15)
        self.assertEqual(15, state.completed_work)
        self.assertEqual(0.15, state.progress)
        self.assertEqual(0.6, state.to_super_work(20))


# test/util/test_temp.py

import os.path
import unittest

from xcube.util.temp import new_temp_dir
from xcube.util.temp import new_temp_file
from xcube.util.temp import remove_dir_later
from xcube.util.temp import remove_file_later


class TempTest(unittest.TestCase):
    def test_new_temp_dir(self):
        dir_path = new_temp_dir()
        self.assertIsInstance(dir_path, str)
        self.assertTrue(os.path.basename(dir_path).startswith('xcube-'))
        self.assertTrue(os.path.isabs(dir_path))
        self.assertTrue(os.path.isdir(dir_path))

        dir_path = new_temp_dir(prefix='bibo.', suffix='.zarr')
        self.assertIsInstance(dir_path, str)
        self.assertTrue(os.path.basename(dir_path).startswith('bibo.'))
        self.assertTrue(os.path.basename(dir_path).endswith('.zarr'))
        self.assertTrue(os.path.isabs(dir_path))
        self.assertTrue(os.path.isdir(dir_path))

    def test_new_temp_file(self):
        fd, file_path = new_temp_file()
        self.assertIsInstance(fd, int)
        self.assertIsInstance(file_path, str)
        self.assertTrue(os.path.basename(file_path).startswith('xcube-'))
        self.assertTrue(os.path.isabs(file_path))
        self.assertTrue(os.path.isfile(file_path))

        fd, file_path = new_temp_file(prefix='bibo.', suffix='.zip')
        self.assertIsInstance(fd, int)
        self.assertIsInstance(file_path, str)
        self.assertTrue(os.path.basename(file_path).startswith('bibo.'))
        self.assertTrue(os.path.basename(file_path).endswith('.zip'))
        self.assertTrue(os.path.isabs(file_path))
        self.assertTrue(os.path.isfile(file_path))

    def test_remove_file_later(self):
        with open('__test__.txt', 'w') as stream:
            stream.write('Hello!')
        file_path = remove_file_later('__test__.txt')
        self.assertTrue(os.path.isabs(file_path))

    def test_remove_dir_later(self):
        os.mkdir('__test__')
        dir_path = remove_dir_later('__test__')
        self.assertTrue(os.path.isabs(dir_path))


# test/util/test_tiledimage.py

from unittest import TestCase

import numpy as np
import PIL.Image

from xcube.util.tiledimage import SourceArrayImage
from xcube.util.tiledimage import ColorMappedRgbaImage
from xcube.util.tiledimage import DirectRgbaImage
from xcube.util.tiledimage import OpImage
from xcube.util.tiledimage import NormalizeArrayImage
from xcube.util.tiledimage import trim_tile


class MyTiledImage(OpImage):
    def __init__(self, size, tile_size):
        super().__init__(size, tile_size, (size[0] // tile_size[0], size[1] // tile_size[1]),
                         mode='int32', format='ndarray')

    def compute_tile(self, tile_x, tile_y, rectangle):
        w, h = self.size
        x, y, tw, th = rectangle
        fill_value = float(x + y * w) / float(w * h)
        return np.full((th, tw), fill_value, np.float32)


class ColorMappedRgbaImageTest(TestCase):
    def test_default(self):
        a = np.linspace(0, 255, 24, dtype=np.int32).reshape((4, 6))
        source_image = SourceArrayImage(a, (2, 2))
        cm_rgb_image = ColorMappedRgbaImage(source_image, format='PNG')
        self.assertEqual(cm_rgb_image.size, (6, 4))
        self.assertEqual(cm_rgb_image.tile_size, (2, 2))
        self.assertEqual(cm_rgb_image.num_tiles, (3, 2))
        tile = cm_rgb_image.get_tile(0, 0)
        self.assertIsInstance(tile, PIL.Image.Image)


class DirectRgbaImageTest(TestCase):
    def test_default(self):
        a = np.linspace(0, 255, 24, dtype=np.int32).reshape((4, 6))
        b = np.linspace(255, 0, 24, dtype=np.int32).reshape((4, 6))
        c = np.linspace(50, 200, 24, dtype=np.int32).reshape((4, 6))
        source_images = [
            SourceArrayImage(a, (2, 2)),
            SourceArrayImage(b, (2, 2)),
            SourceArrayImage(c, (2, 2)),
        ]
        cm_rgb_image = DirectRgbaImage(source_images, format='PNG')
        self.assertEqual(cm_rgb_image.size, (6, 4))
        self.assertEqual(cm_rgb_image.tile_size, (2, 2))
        self.assertEqual(cm_rgb_image.num_tiles, (3, 2))
        tile = cm_rgb_image.get_tile(0, 0)
        self.assertIsInstance(tile, PIL.Image.Image)


class NormalizeArrayImageTest(TestCase):
    def test_default(self):
        a = np.arange(0, 24, dtype=np.int32).reshape((4, 6))
        source_image = SourceArrayImage(a, (2, 2))
        target_image = NormalizeArrayImage(source_image)

        self.assertEqual(target_image.size, (6, 4))
        self.assertEqual(target_image.tile_size, (2, 2))
        self.assertEqual(target_image.num_tiles, (3, 2))

        self.assertEqual(target_image.get_tile(0, 0).tolist(), [[0, 1],
                                                                [6, 7]])
        self.assertEqual(target_image.get_tile(1, 0).tolist(), [[2, 3],
                                                                [8, 9]])
        self.assertEqual(target_image.get_tile(2, 0).tolist(), [[4, 5],
                                                                [10, 11]])

        self.assertEqual(target_image.get_tile(0, 1).tolist(), [[12, 13],
                                                                [18, 19]])
        self.assertEqual(target_image.get_tile(1, 1).tolist(), [[14, 15],
                                                                [20, 21]])
        self.assertEqual(target_image.get_tile(2, 1).tolist(), [[16, 17],
                                                                [22, 23]])

    def test_force_2d(self):
        a = np.arange(0, 48, dtype=np.int32).reshape((2, 4, 6))
        source_image = SourceArrayImage(a[0], (2, 2))
        target_image = NormalizeArrayImage(source_image, force_2d=True)

        self.assertEqual(target_image.size, (6, 4))
        self.assertEqual(target_image.tile_size, (2, 2))
        self.assertEqual(target_image.num_tiles, (3, 2))


class SourceArrayImageTest(TestCase):

    def test_flip_y_tiled_evenly(self):
        a = np.arange(0, 24, dtype=np.int32).reshape((4, 6))
        source_array_image = SourceArrayImage(a, (2, 2), flip_y=True)

        self.assertEqual(source_array_image.size, (6, 4))
        self.assertEqual(source_array_image.tile_size, (2, 2))
        self.assertEqual(source_array_image.num_tiles, (3, 2))

        self.assertEqual(source_array_image.get_tile(0, 0).tolist(),
                         [[18, 19],
                          [12, 13]])
        self.assertEqual(source_array_image.get_tile(1, 0).tolist(),
                         [[20, 21],
                          [14, 15]])
        self.assertEqual(source_array_image.get_tile(2, 0).tolist(),
                         [[22, 23],
                          [16, 17]])

        self.assertEqual(source_array_image.get_tile(0, 1).tolist(),
                         [[6, 7],
                          [0, 1]])
        self.assertEqual(source_array_image.get_tile(1, 1).tolist(),
                         [[8, 9],
                          [2, 3]])
        self.assertEqual(source_array_image.get_tile(2, 1).tolist(),
                         [[10, 11],
                          [4, 5]])

    def test_flip_y_not_tiled_evenly(self):
        a = np.arange(0, 30, dtype=np.int32).reshape((5, 6))
        source_array_image = SourceArrayImage(a, (2, 2), flip_y=True)

        self.assertEqual(source_array_image.size, (6, 5))
        self.assertEqual(source_array_image.tile_size, (2, 2))
        self.assertEqual(source_array_image.num_tiles, (3, 3))

        self.assertEqual(source_array_image.get_tile(0, 0).tolist(),
                         [[24, 25],
                          [18, 19]])
        self.assertEqual(source_array_image.get_tile(1, 0).tolist(),
                         [[26, 27],
                          [20, 21]])
        self.assertEqual(source_array_image.get_tile(2, 0).tolist(),
                         [[28, 29],
                          [22, 23]])

        self.assertEqual(source_array_image.get_tile(0, 1).tolist(),
                         [[12, 13],
                          [6, 7]])
        self.assertEqual(source_array_image.get_tile(1, 1).tolist(),
                         [[14, 15],
                          [8, 9]])
        self.assertEqual(source_array_image.get_tile(2, 1).tolist(),
                         [[16, 17],
                          [10, 11]])

        tile_0_2 = source_array_image.get_tile(0, 2).tolist()
        self.assertEqual(tile_0_2[0], [0, 1])
        self.assertTrue(np.isnan(tile_0_2[1][0]))
        self.assertTrue(np.isnan(tile_0_2[1][1]))
        tile_1_2 = source_array_image.get_tile(1, 2).tolist()
        self.assertEqual(tile_1_2[0], [2, 3])
        self.assertTrue(tile_1_2[1][0])
        self.assertTrue(tile_1_2[1][1])
        tile_2_2 = source_array_image.get_tile(2, 2).tolist()
        self.assertEqual(tile_2_2[0], [4, 5])
        self.assertTrue(tile_2_2[1][0])
        self.assertTrue(tile_2_2[1][1])


class TrimTileTest(TestCase):
    # noinspection PyMethodMayBeStatic
    def test_trim_tile(self):
        a = np.arange(0, 6, dtype=np.float32)
        a.shape = 2, 3
        b = trim_tile(a, (3, 2))
        np.testing.assert_equal(b, np.array([[0., 1., 2.],
                                             [3., 4., 5.]]))
        b = trim_tile(a, (4, 2))
        np.testing.assert_equal(b, np.array([[0., 1., 2., np.nan],
                                             [3., 4., 5., np.nan]]))
        b = trim_tile(a, (3, 3))
        np.testing.assert_equal(b, np.array([[0., 1., 2.],
                                             [3., 4., 5.],
                                             [np.nan, np.nan, np.nan]]))
        b = trim_tile(a, (4, 3))
        np.testing.assert_equal(b, np.array([[0., 1., 2., np.nan],
                                             [3., 4., 5., np.nan],
                                             [np.nan, np.nan, np.nan, np.nan]]))


# test/util/test_tilegrid.py

import unittest
from unittest import TestCase

import pyproj

from xcube.util.tilegrid import GeographicTileGrid
from xcube.util.tilegrid import ImageTileGrid
from xcube.util.tilegrid import tile_grid_to_ol_xyz_source_options


class Crs84TileGridTest(TestCase):
    def test_it(self):
        tg = GeographicTileGrid()
        self.assertEqual((256, 256), tg.tile_size)
        self.assertEqual(None, tg.min_level)
        self.assertEqual(None, tg.max_level)
        self.assertEqual(pyproj.CRS.from_string('CRS84'), tg.crs)
        self.assertEqual((-180, -90, 180, 90), tg.extent)
        self.assertEqual((-180, 90), tg.origin)
        self.assertEqual(False, tg.is_j_axis_up)
        self.assertEqual((512, 256), tg.get_image_size(0))
        self.assertEqual((1024, 512), tg.get_image_size(1))
        self.assertEqual((512 * 2 ** 15, 256 * 2 ** 15), tg.get_image_size(15))
        self.assertEqual((2, 1), tg.get_num_tiles(0))
        self.assertEqual((4, 2), tg.get_num_tiles(1))
        self.assertEqual((2 ** 16, 2 ** 15), tg.get_num_tiles(15))
        self.assertEqual(180 / 256, tg.get_resolution(0))
        self.assertEqual(180 / 512, tg.get_resolution(1))
        self.assertEqual(180 / (256 * 2 ** 15), tg.get_resolution(15))


class ImageTileGridTest(TestCase):
    def test_it(self):
        tg = ImageTileGrid(image_size=(4000, 2000),
                           tile_size=(512, 512),
                           crs=pyproj.CRS.from_string('WGS84'),
                           xy_res=0.05,
                           xy_min=(0, 0),
                           is_j_axis_up=True)
        self.assertEqual((512, 512), tg.tile_size)
        self.assertEqual(None, tg.min_level)
        self.assertEqual(2, tg.max_level)
        self.assertEqual(pyproj.CRS.from_string('WGS84'), tg.crs)
        self.assertEqual((0, 0, 200.0, 100.0), tg.extent)
        self.assertEqual((0, 100.0), tg.origin)
        self.assertEqual(True, tg.is_j_axis_up)
        self.assertEqual((1000, 500), tg.get_image_size(0))
        self.assertEqual((2000, 1000), tg.get_image_size(1))
        self.assertEqual((4000, 2000), tg.get_image_size(2))
        self.assertEqual((2, 1), tg.get_num_tiles(0))
        self.assertEqual((4, 2), tg.get_num_tiles(1))
        self.assertEqual((8, 4), tg.get_num_tiles(2))
        self.assertEqual(0.2, tg.get_resolution(0))
        self.assertEqual(0.1, tg.get_resolution(1))
        self.assertEqual(0.05, tg.get_resolution(2))


class OpenLayersOptionsTest(unittest.TestCase):
    def test_geographic_only(self):
        tile_grid = ImageTileGrid(image_size=(7200, 3600),
                                  tile_size=(512, 512),
                                  crs='EPSG:32633',
                                  xy_res=180 / 3600,
                                  xy_min=(-180, -90),
                                  is_j_axis_up=True)
        self.assertIsNone(tile_grid_to_ol_xyz_source_options(
            tile_grid,
            'https://xcube.server/tile/{x}/{y}/{z}.png'
        ))

    def test_with_max_level(self):
        tile_grid = ImageTileGrid(image_size=(7200, 3600),
                                  tile_size=(512, 512),
                                  crs='EPSG:4326',
                                  xy_res=180. / 3600,
                                  xy_min=(-180., -90.),
                                  is_j_axis_up=True)
        self.assertEqual(
            {
                'url': 'https://xcube.server/tile/{x}/{y}/{z}.png',
                'projection': 'EPSG:4326',
                'tileGrid': {
                    'tileSize': [512, 512],
                    'sizes': [[2, 1], [4, 2], [8, 4], [15, 8]],
                    'resolutions': [0.4, 0.2, 0.1, 0.05],
                    'extent': [-180., -90., 180., 90.],
                    'origin': [-180., 90.],
                },
            },
            tile_grid_to_ol_xyz_source_options(
                tile_grid,
                'https://xcube.server/tile/{x}/{y}/{z}.png'
            )
        )

    def test_with_wo_max_level(self):
        tile_grid = GeographicTileGrid()
        self.assertEqual(
            {
                'url': 'https://xcube.server/tile/{x}/{y}/{z}.png',
                'projection': 'EPSG:4326',
                'tileSize': [256, 256],
                'maxResolution': 0.703125,
            },
            tile_grid_to_ol_xyz_source_options(
                tile_grid,
                'https://xcube.server/tile/{x}/{y}/{z}.png'
            )
        )


# test/util/test_tilegrid2.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.


import unittest

from xcube.util.tilegrid2 import EARTH_CIRCUMFERENCE_WGS84
from xcube.util.tilegrid2 import TileGrid2


class TileGridTest(unittest.TestCase):

    def test_web_mercator(self):
        tile_grid = TileGrid2.new_web_mercator()

        self.assertEqual((1, 1), tile_grid.num_level_zero_tiles)
        self.assertEqual('EPSG:3857', tile_grid.crs_name)
        self.assertEqual(256, tile_grid.tile_size)
        self.assertEqual(EARTH_CIRCUMFERENCE_WGS84, tile_grid.map_height)
        self.assertEqual(None, tile_grid.map_levels)
        self.assertEqual(None, tile_grid.map_resolutions)

    def test_web_mercator_bbox(self):
        tile_grid = TileGrid2.new_web_mercator()

        half = EARTH_CIRCUMFERENCE_WGS84 / 2

        self.assertEqual((-half, -half, half, half),
                         tile_grid.get_tile_bbox(0, 0, 0))

        self.assertEqual((-half, 0.0, 0.0, half),
                         tile_grid.get_tile_bbox(0, 0, 1))
        self.assertEqual((0.0, 0.0, half, half),
                         tile_grid.get_tile_bbox(1, 0, 1))
        self.assertEqual((-half, -half, 0.0, 0.0),
                         tile_grid.get_tile_bbox(0, 1, 1))
        self.assertEqual((0.0, -half, half, 0.0),
                         tile_grid.get_tile_bbox(1, 1, 1))

    def test_geographic(self):
        tile_grid = TileGrid2.new_geographic()

        self.assertEqual((2, 1), tile_grid.num_level_zero_tiles)
        self.assertEqual('EPSG:4326', tile_grid.crs_name)
        self.assertEqual(256, tile_grid.tile_size)
        self.assertEqual(180, tile_grid.map_height)
        self.assertEqual(None, tile_grid.map_levels)
        self.assertEqual(None, tile_grid.map_resolutions)

    def test_geographic_bbox(self):
        tile_grid = TileGrid2.new_geographic()

        self.assertEqual((-180, -90, 0, 90), tile_grid.get_tile_bbox(0, 0, 0))
        self.assertEqual((0, -90, 180, 90), tile_grid.get_tile_bbox(1, 0, 0))

        self.assertEqual((-180, 0, -90, 90), tile_grid.get_tile_bbox(0, 0, 1))
        self.assertEqual((-90, 0, 0, 90), tile_grid.get_tile_bbox(1, 0, 1))
        self.assertEqual((0, 0, 90, 90), tile_grid.get_tile_bbox(2, 0, 1))
        self.assertEqual((90, 0, 180, 90), tile_grid.get_tile_bbox(3, 0, 1))
        self.assertEqual((-180, -90, -90, 0),
                         tile_grid.get_tile_bbox(0, 1, 1))
        self.assertEqual((-90, -90, 0, 0), tile_grid.get_tile_bbox(1, 1, 1))
        self.assertEqual((0, -90, 90, 0), tile_grid.get_tile_bbox(2, 1, 1))
        self.assertEqual((90, -90, 180, 0), tile_grid.get_tile_bbox(3, 1, 1))

    def test_geographic_dataset_level(self):
        tile_grid = TileGrid2.new_web_mercator()

        for level, res in enumerate(tile_grid.resolutions('meter')):
            print(level, res)
            if level > 16:
                break

        args = [10, 20, 40, 80, 160], 'meter'

        self.assertEqual(4, tile_grid.get_dataset_level(0, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(1, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(2, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(3, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(4, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(5, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(6, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(7, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(8, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(9, *args))
        self.assertEqual(4, tile_grid.get_dataset_level(10, *args))
        self.assertEqual(3, tile_grid.get_dataset_level(11, *args))
        self.assertEqual(2, tile_grid.get_dataset_level(12, *args))
        self.assertEqual(1, tile_grid.get_dataset_level(13, *args))
        self.assertEqual(0, tile_grid.get_dataset_level(14, *args))
        self.assertEqual(0, tile_grid.get_dataset_level(15, *args))
        self.assertEqual(0, tile_grid.get_dataset_level(16, *args))
        self.assertEqual(0, tile_grid.get_dataset_level(17, *args))
        self.assertEqual(0, tile_grid.get_dataset_level(18, *args))
        self.assertEqual(0, tile_grid.get_dataset_level(19, *args))


# test/util/test_types.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import unittest

from xcube.util.types import normalize_scalar_or_pair


class NormalizeNumberScalarOrPairTest(unittest.TestCase):

    def test_scalar_float(self):
        self.assertEqual((1.1, 1.1),
                         normalize_scalar_or_pair(1.1,
                                                  item_type=float))
        self.assertEqual((2.0, 2.0),
                         normalize_scalar_or_pair(2.0,
                                                  item_type=float))

    def test_scalar_int(self):
        self.assertEqual((1, 1), normalize_scalar_or_pair(1, item_type=int))
        self.assertEqual((1, 1), normalize_scalar_or_pair(1, item_type=int))
        self.assertEqual((2, 2), normalize_scalar_or_pair(2, item_type=int))

    def test_pair_float(self):
        self.assertEqual((1.1, 1.9),
                         normalize_scalar_or_pair((1.1, 1.9),
                                                  item_type=float))
        self.assertEqual((2.0, 1.9),
                         normalize_scalar_or_pair((2.0, 1.9),
                                                  item_type=float))

    def test_pair_int(self):
        self.assertEqual((1, 1),
                         normalize_scalar_or_pair((1, 1), item_type=int))
        self.assertEqual((2, 1),
                         normalize_scalar_or_pair((2, 1), item_type=int))

    def test_wrong_type(self):
        with self.assertRaises(ValueError) as ve:
            normalize_scalar_or_pair('xyz',
                                     item_type=int,
                                     name='notastring')
        self.assertEqual(
            "notastring must be a scalar or pair of <class 'int'>, was 'xyz'",
            f'{ve.exception}'
        )

    def test_wrong_length(self):
        with self.assertRaises(ValueError) as ve:
            normalize_scalar_or_pair((0, 1, 2))
        self.assertEqual(
            "Value must be a scalar or pair of scalars, was '(0, 1, 2)'",
            f'{ve.exception}'
        )

# test/util/test_undefined.py

import copy
from unittest import TestCase

from xcube.util.undefined import UNDEFINED


class UndefinedTest(TestCase):
    def test_it(self):
        self.assertIsNotNone(UNDEFINED)
        self.assertEqual(str(UNDEFINED), 'UNDEFINED')
        self.assertEqual(repr(UNDEFINED), 'UNDEFINED')
        undefined_copy = copy.deepcopy(UNDEFINED)
        self.assertEqual(UNDEFINED, undefined_copy)
        self.assertEqual(hash(UNDEFINED), hash(undefined_copy))


# test/util/test_versions.py

import unittest

import xcube.version as xcube_version
from xcube.util.versions import get_xcube_versions


class GetVersionsTest(unittest.TestCase):

    def test_it_contains_what_is_expected(self):
        versions = get_xcube_versions()
        self.assertIsInstance(versions, dict)
        self.assertTrue(all(isinstance(k, str) for k in versions.keys()))
        self.assertTrue(all(isinstance(v, str) for v in versions.values()))
        self.assertIn('xarray', versions)
        self.assertIn('dask', versions)
        self.assertIn('zarr', versions)
        self.assertIn('tornado', versions)
        self.assertIn('pandas', versions)
        self.assertIn('numpy', versions)
        self.assertIn('geopandas', versions)
        self.assertIn('xcube', versions)
        self.assertEqual(xcube_version, versions['xcube'])

    def test_it_is_cached(self):
        versions = get_xcube_versions()
        self.assertIs(versions, get_xcube_versions())


# setup.py

#!/usr/bin/env python3

# The MIT License (MIT)
# Copyright (c) 2021 by Brockmann Consult GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.


from setuptools import setup, find_packages

requirements = [
    #
    # xcube requirements are given in file ./environment.yml.
    #
    # All packages here have been commented out, because otherwise setuptools will install
    # additional pip packages although conda packages with same name are already available
    # in the conda environment defined by file ./environment.yml.
    #
    # 'affine',
    # 'click',
    # 'cmocean',
    # 'dask',
    # 'fiona',
    # 'gdal',
    # 'matplotlib',
    # 'netcdf4',
    # 'numba',
    # 'numpy',
    # 'pandas',
    # 'pillow',
    # 'proj4',
    # 'pyyaml',
    # 'rasterio',
    # 's3fs',
    # 'setuptools',
    # 'shapely',
    # 'tornado',
    # 'xarray',
    # 'zarr',
]

packages = find_packages(exclude=["test", "test.*"])

# Same effect as "from cate import version", but avoids importing cate:
version = None
with open('xcube/version.py') as f:
    exec(f.read())

setup(
    name="xcube",
    version=version,
    description=('xcube is a Python package for generating and exploiting '
                 'data cubes powered by xarray, dask, and zarr.'),
    license='MIT',
    author='xcube Development Team',
    packages=packages,
    entry_points={
        'console_scripts': [
            # xcube's CLI
            'xcube = xcube.cli.main:main',
        ],
        'xcube_plugins': [
            # xcube's default extensions
            'xcube = xcube.plugin:init_plugin',
        ],
    },
    install_requires=requirements,
)


# test/__init__.py

from test.util.test_plugin import init_plugin

# xcube.util.plugin:discover_plugin_modules() will search for "init_test" first (because this package is called "test").
# Only if "init_test" is not found it would search for "init_plugin".
init_test = init_plugin


# test/mixins.py

from typing import Any
import array

# noinspection PyUnresolvedReferences
class AlmostEqualDeepMixin:

    def assertAlmostEqualDeep(self,
                              first: Any,
                              second: Any,
                              msg: str = None,
                              places: int = None,
                              delta: float = None):
        """
        Performs self.assertAlmostEqual() on number, and numbers in dictionaries and sequences.

        Note that decimal places (from zero) are usually not the same
        as significant digits (measured from the most significant digit).

        If the two objects compare equal then they will automatically
        compare almost equal.
        """
        if first == second:
            # shortcut
            return

        if isinstance(first, float) or isinstance(second, float):
            self.assertAlmostEqual(first, second, msg=msg, places=places, delta=delta)
            # success
            return

        if isinstance(first, set) or isinstance(second, set):
            # we currently cannot almost-equal-compare sets, but it is possible:
            #  - first remove items from both sets that are strictly equal
            #  - secondly remove items from both sets that are almost equal
            #  - are both empty in the end?
            self.assertEqual(first, second, msg=msg)
            # success
            return

        if isinstance(first, dict) and isinstance(second, dict):
            if len(first) != len(second):
                self.assertEqual(first, second, msg=msg)  # fail
            for key, first_value in first.items():
                if key not in second:
                    self.assertEqual(first, second, msg=msg)  # fail
                second_value = second[key]
                self.assertAlmostEqualDeep(first_value, second_value, msg=msg, delta=delta, places=places)
            for key, second_value in second.items():
                if key not in first:
                    self.assertEqual(first, second, msg=msg)  # fail
            # success
            return

        if isinstance(first, (list, tuple, array.array)) and isinstance(second, (list, tuple, array.array)):
            if len(first) != len(second):
                # let fail
                self.assertEqual(first, second, msg=msg)
            for first_value, second_value in zip(first, second):
                self.assertAlmostEqualDeep(first_value, second_value, msg=msg, delta=delta, places=places)
            # success
            return

        self.assertEqual(first, second, msg=msg)


# test/plugin.py

# init_plugin() is used by test/util/test_plugin

# noinspection PyUnresolvedReferences
from test.util.test_plugin import init_plugin

# test/s3test.py

import os
import subprocess
import sys
import time
import unittest
import urllib
import urllib.error
import urllib.request

import moto.server

MOTO_SERVER_ENDPOINT_URL = f'http://127.0.0.1:5000'

MOTOSERVER_PATH = moto.server.__file__
MOTOSERVER_ARGS = [sys.executable, MOTOSERVER_PATH, 's3']


class S3Test(unittest.TestCase):
    _moto_server = None

    @classmethod
    def setUpClass(cls) -> None:

        """Mocked AWS Credentials for moto."""
        os.environ['AWS_ACCESS_KEY_ID'] = 'testing'
        os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing'
        os.environ['AWS_SECURITY_TOKEN'] = 'testing'
        os.environ['AWS_SESSION_TOKEN'] = 'testing'

        cls._moto_server = subprocess.Popen(MOTOSERVER_ARGS)
        t0 = time.perf_counter()
        running = False
        while not running and time.perf_counter() - t0 < 60.0:
            try:
                with urllib.request.urlopen(MOTO_SERVER_ENDPOINT_URL, timeout=1.0):
                    running = True
                    print(f'moto_server started after {round(1000 * (time.perf_counter() - t0))} ms')

            except urllib.error.URLError as e:
                pass
        if not running:
            raise Exception(f'Failed to start moto server after {round(1000 * (time.perf_counter() - t0))} ms')

    def setUp(self) -> None:
        # see https://github.com/spulec/moto/issues/2288
        urllib.request.urlopen(urllib.request.Request(MOTO_SERVER_ENDPOINT_URL + '/moto-api/reset', method='POST'))

    @classmethod
    def tearDownClass(cls) -> None:
        cls._moto_server.kill()


# test/sampledata.py

import numpy as np
import pandas as pd
import xarray as xr


def new_test_dataset(time, height=180, **indexers):
    """
    Create a test dataset with dimensions ("time", "lat", "lon") and data variables given by *indexers*.

    :param time: Single date/time string or sequence of date-time strings.
    :param height: Size of the latitude dimension.
    :param indexers: Variable name to value mapping.
           Value may be a scalar or a vector of same length as *time*.
    :return: test dataset
    """
    # TODO (forman): get rid of this code here, utilise xcube.core.new_cube() instead
    time = [time] if isinstance(time, str) else time
    width = height * 2
    num_times = len(time)
    res = 180 / height
    shape = (1, height, width)
    data_vars = dict()
    for name, value in indexers.items():
        try:
            values = list(value)
        except TypeError:
            values = [value] * num_times
        if len(values) != num_times:
            raise ValueError()
        data_vars[name] = (['time', 'lat', 'lon'],
                           np.concatenate(tuple(np.full(shape, values[i]) for i in range(num_times))))
    time = np.array(pd.to_datetime(time), dtype=np.datetime64)
    return xr.Dataset(data_vars,
                      coords=dict(time=(
                          ['time'], time, dict(units='nanoseconds since 1970-01-01', calendar='proleptic_gregorian')),
                          lat=(['lat'], np.linspace(-90 + res, +90 - res, height)),
                          lon=(['lon'], np.linspace(-180 + res, +180 - res, width))))


def create_s2plus_dataset():
    x = xr.DataArray([310005., 310015., 310025., 310035., 310045.], dims=["x"],
                     attrs=dict(units="m", standard_name="projection_x_coordinate"))
    y = xr.DataArray([5689995., 5689985., 5689975., 5689965., 5689955.], dims=["y"],
                     attrs=dict(units="m", standard_name="projection_y_coordinate"))
    lon = xr.DataArray([[0.272763, 0.272906, 0.273050, 0.273193, 0.273336],
                        [0.272768, 0.272911, 0.273055, 0.273198, 0.273342],
                        [0.272773, 0.272917, 0.273060, 0.273204, 0.273347],
                        [0.272779, 0.272922, 0.273066, 0.273209, 0.273352],
                        [0.272784, 0.272927, 0.273071, 0.273214, 0.273358]],
                       dims=["y", "x"], attrs=dict(units="degrees_east", standard_name="longitude"))
    lat = xr.DataArray([[51.329464, 51.329464, 51.329468, 51.32947, 51.329475],
                        [51.329372, 51.329376, 51.32938, 51.329384, 51.329388],
                        [51.329285, 51.329285, 51.32929, 51.329292, 51.329296],
                        [51.329193, 51.329197, 51.32920, 51.329205, 51.329205],
                        [51.329100, 51.329105, 51.32911, 51.329113, 51.329117]],
                       dims=["y", "x"], attrs=dict(units="degrees_north", standard_name="latitude"))
    rrs_443 = xr.DataArray([[0.014000, 0.014000, 0.016998, 0.016998, 0.016998],
                            [0.014000, 0.014000, 0.016998, 0.016998, 0.016998],
                            [0.019001, 0.019001, 0.016998, 0.016998, 0.016998],
                            [0.019001, 0.019001, 0.016998, 0.016998, 0.016998],
                            [0.019001, 0.019001, 0.016998, 0.016998, 0.016998]],
                           dims=["y", "x"], attrs=dict(units="sr-1", grid_mapping="transverse_mercator"))
    rrs_665 = xr.DataArray([[0.025002, 0.019001, 0.008999, 0.012001, 0.022999],
                            [0.028000, 0.021000, 0.009998, 0.008999, 0.022999],
                            [0.036999, 0.022999, 0.007999, 0.008999, 0.023998],
                            [0.041000, 0.022999, 0.007000, 0.009998, 0.021000],
                            [0.033001, 0.018002, 0.007999, 0.008999, 0.021000]],
                           dims=["y", "x"], attrs=dict(units="sr-1", grid_mapping="transverse_mercator"))
    transverse_mercator = xr.DataArray(np.array([0xffffffff], dtype=np.uint32),
                                       attrs=dict(grid_mapping_name="transverse_mercator",
                                                  scale_factor_at_central_meridian=0.9996,
                                                  longitude_of_central_meridian=3.0,
                                                  latitude_of_projection_origin=0.0,
                                                  false_easting=500000.0,
                                                  false_northing=0.0,
                                                  semi_major_axis=6378137.0,
                                                  inverse_flattening=298.257223563))
    return xr.Dataset(dict(rrs_443=rrs_443, rrs_665=rrs_665, transverse_mercator=transverse_mercator),
                      coords=dict(x=x, y=y, lon=lon, lat=lat),
                      attrs={
                          "title": "T31UCS_20180802T105621",
                          "conventions": "CF-1.6",
                          "institution": "VITO",
                          "product_type": "DCS4COP Sentinel2 Product",
                          "origin": "Copernicus Sentinel Data",
                          "project": "DCS4COP",
                          "time_coverage_start": "2018-08-02T10:59:38.888000Z",
                          "time_coverage_end": "2018-08-02T10:59:38.888000Z"
                      })


def create_highroc_dataset(no_spectra=False):
    """
    Simulates a HIGHROC OLCI L2 product in NetCDF 4 format
    """
    lon = np.array([[8, 9.3, 10.6, 11.9],
                    [8, 9.2, 10.4, 11.6],
                    [8, 9.1, 10.2, 11.3]], dtype=np.float32)
    lat = np.array([[56, 56.1, 56.2, 56.3],
                    [55, 55.2, 55.4, 55.6],
                    [54, 54.3, 54.6, 54.9]], dtype=np.float32)

    if not no_spectra:
        wavelengths = [(1, 400.0), (2, 412.5), (3, 442.5), (4, 490.0), (5, 510.0),
                       (6, 560.0), (7, 620.0), (8, 665.0), (9, 673.75), (10, 681.25),
                       (11, 708.75), (12, 753.75), (16, 778.75), (17, 865.0), (18, 885.0), (21, 940.0)]
        rtoa_desc = "Top-of-atmosphere reflectance"
        rrs_desc = "Atmospherically corrected angular dependent remote sensing reflectances"
        rtoa_vars = {f'rtoa_{i}': create_waveband(i, wl, '1', rtoa_desc) for i, wl in wavelengths}
        rrs_vars = {f'rrs_{i}': create_waveband(i, wl, 'sr^-1', rrs_desc) for i, wl in wavelengths}
    else:
        rtoa_vars = {}
        rrs_vars = {}

    return xr.Dataset(
        data_vars=dict(
            conc_chl=create_conc_chl(),
            c2rcc_flags=create_c2rcc_flag_var(),
            lon=(('y', 'x'), lon, dict(
                long_name="longitude",
                units="degrees_east",
            )),
            lat=(('y', 'x'), lat, dict(
                long_name="latitude",
                units="degrees_north",
            )),
            **rtoa_vars,
            **rrs_vars,
        ),
        attrs=dict(start_date='14-APR-2017 10:27:50.183264',
                   stop_date='14-APR-2017 10:31:42.736226')
    )


def create_waveband(index, wavelength, units, long_name=None):
    data = np.array([[7, 11, np.nan, 5],
                     [5, 10, 2, 21],
                     [16, 6, 20, 17]], dtype=np.float32)
    return (('y', 'x'), data, dict(
        long_name=long_name,
        units=units,
        spectral_band_index=index,
        wavelength=wavelength,
        bandwidth=15.0,
        valid_pixel_expression="c2rcc_flags.F1",
        _FillValue=np.nan,
    ))


def create_conc_chl():
    data = np.array([[7, 11, np.nan, 5],
                     [5, 10, 2, 21],
                     [16, 6, 20, 17]], dtype=np.float32)
    return (('y', 'x'), data, dict(
        long_name="Chlorophylll concentration",
        units="mg m^-3",
        _FillValue=np.nan,
        valid_pixel_expression="c2rcc_flags.F1",
    ))


def create_c2rcc_flag_var():
    data = np.array([[1, 1, 1, 1],
                     [1, 4, 1, 2],
                     [8, 1, 1, 1]], dtype=np.uint32)
    return xr.DataArray(data, dims=('y', 'x'), name='c2rcc_flags', attrs=dict(
        long_name="C2RCC quality flags",
        _Unsigned="true",
        flag_meanings="F1 F2 F3 F4",
        flag_masks=np.array([1, 2, 4, 8], np.int32),
        flag_coding_name="c2rcc_flags",
        flag_descriptions="D1 D2 D3 D4",
    ))


def create_cmems_sst_flag_var():
    sea = 1
    land = 2
    lake = 4
    ice = 8
    data = np.array([[[sea + ice, land + ice, lake + ice, lake],
                      [sea + ice, sea, land, land],
                      [sea, sea, sea, land]]], dtype=np.float32)
    return xr.DataArray(data, dims=('time', 'lat', 'lon'), name='mask', attrs=dict(
        long_name="land sea ice lake bit mask",
        flag_masks="0b, 1b, 2b, 3b",
        flag_meanings="sea land lake ice",
        valid_min=0,
        valid_max=12,
    ))


class SourceDatasetMixin:

    @classmethod
    def new_2x2_dataset_with_irregular_coords(cls):
        lon = np.array([[1.0, 6.0],
                        [0.0, 2.0]])
        lat = np.array([[56.0, 53.0],
                        [52.0, 50.0]])
        rad = np.array([[1.0, 2.0],
                        [3.0, 4.0]])
        return xr.Dataset(dict(lon=xr.DataArray(lon, dims=('y', 'x')),
                               lat=xr.DataArray(lat, dims=('y', 'x')),
                               rad=xr.DataArray(rad, dims=('y', 'x'))))

    @classmethod
    def new_2x2_dataset_with_irregular_coords_antimeridian(cls):
        lon = np.array([[+179.0, -176.0],
                        [+178.0, +180.0]])
        lat = np.array([[56.0, 53.0],
                        [52.0, 50.0]])
        rad = np.array([[1.0, 2.0],
                        [3.0, 4.0]])
        return xr.Dataset(dict(lon=xr.DataArray(lon, dims=('y', 'x')),
                               lat=xr.DataArray(lat, dims=('y', 'x')),
                               rad=xr.DataArray(rad, dims=('y', 'x'))))


# test/test_mixins.py

import unittest

from test.mixins import AlmostEqualDeepMixin


class AlmostEqualDeepMixinTest(unittest.TestCase, AlmostEqualDeepMixin):
    def test_int_and_float_7_places_default(self):
        self.assertAlmostEqualDeep(0, 0.8e-8)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(0, 0.8e-7)

    def test_int(self):
        self.assertAlmostEqualDeep(45, 45)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(45, 54)

    def test_str(self):
        self.assertAlmostEqualDeep("abc", "abc")
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep("abc", "Abc")

    def test_bool(self):
        self.assertAlmostEqualDeep(True, True)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(True, False)

    def test_set(self):
        expected = {'a', 1.1256, True}
        self.assertAlmostEqualDeep(expected, expected)
        self.assertAlmostEqualDeep(expected, {'a', 1.1256, True})
        with self.assertRaises(AssertionError):
            # We currently don't test sets
            self.assertAlmostEqualDeep(expected, {'a', 1.1251, True}, places=2)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, {'a', 1.1256, False})

    def test_dict(self):
        expected = {'a': 1.1256, 'b': 5}
        self.assertAlmostEqualDeep(expected, expected)
        self.assertAlmostEqualDeep(expected, {'a': 1.1256, 'b': 5})
        self.assertAlmostEqualDeep(expected, {'a': 1.1251, 'b': 5}, places=3)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, {'a': 1.1251, 'b': 5}, places=4)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, {'a': 1.1256, 'b': 6})
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, [1, 2, 3])
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, 3456)

    def test_list(self):
        expected = ['a', 1.1256, True]
        self.assertAlmostEqualDeep(expected, expected)
        self.assertAlmostEqualDeep(expected, ['a', 1.1256, True])
        self.assertAlmostEqualDeep(expected, ('a', 1.1256, True))
        self.assertAlmostEqualDeep(expected, ['a', 1.1251, True], places=3)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, ['a', 1.1251, True], places=4)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, ['a', 1.1256, False], places=4)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, [1, 2, 3])
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, 3456)

    def test_list_dict_tuple(self):
        expected = [
            {'a': True, 'b': (1.1256, 45, True)},
            {'a': False, 'b': (2.1256, 46, False)}
        ]
        self.assertAlmostEqualDeep(expected, expected)
        self.assertAlmostEqualDeep(expected, [
            {'a': True, 'b': (1.1256, 45, True)},
            {'a': False, 'b': (2.1256, 46, False)}
        ])
        self.assertAlmostEqualDeep(expected, [
            {'a': True, 'b': (1.1251, 45, True)},
            {'a': False, 'b': (2.1259, 46, False)}
        ], places=3)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, [
                {'a': True, 'b': (1.1251, 45, True)},
                {'a': False, 'b': (2.1259, 46, False)}
            ], places=4)


# test/test_plugin.py

import unittest

from xcube.plugin import init_plugin
from xcube.util.extension import ExtensionRegistry


class PluginTest(unittest.TestCase):
    def test_standard_extensions_exist(self):
        ext_reg = ExtensionRegistry()

        init_plugin(ext_reg)

        self.assertTrue(ext_reg.has_extension('xcube.core.gen.iproc', 'default'))

        self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'zarr'))
        self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'netcdf4'))
        self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'csv'))
        self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'mem'))

        self.assertTrue(ext_reg.has_extension('xcube.core.store', 'file'))
        self.assertTrue(ext_reg.has_extension('xcube.core.store', 'memory'))
        self.assertTrue(ext_reg.has_extension('xcube.core.store', 's3'))

        self.assertTrue(ext_reg.has_extension('xcube.cli', 'compute'))
        self.assertTrue(ext_reg.has_extension('xcube.cli', 'extract'))
        self.assertTrue(ext_reg.has_extension('xcube.cli', 'gen'))
        self.assertTrue(ext_reg.has_extension('xcube.cli', 'level'))
        self.assertTrue(ext_reg.has_extension('xcube.cli', 'optimize'))



# test/webapi/helpers.py

import os
from typing import Optional, Dict, Mapping

import yaml

from xcube.util.undefined import UNDEFINED
from xcube.webapi.context import ServiceContext, MultiLevelDatasetOpener
from xcube.webapi.errors import ServiceBadRequestError
from xcube.webapi.reqparams import RequestParams


def new_test_service_context(config_file_name: str = 'config.yml',
                             ml_dataset_openers: Dict[str, MultiLevelDatasetOpener] = None,
                             prefix: str = None) -> ServiceContext:
    ctx = ServiceContext(base_dir=get_res_test_dir(), ml_dataset_openers=ml_dataset_openers, prefix=prefix)
    config_file = os.path.join(ctx.base_dir, config_file_name)
    with open(config_file, encoding='utf-8') as fp:
        ctx.config = yaml.safe_load(fp)
    return ctx


def get_res_test_dir() -> str:
    return os.path.normpath(os.path.join(os.path.dirname(__file__), 'res', 'test'))


def get_res_demo_dir() -> str:
    return os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..', 'xcube', 'webapi', 'res', 'demo'))


class RequestParamsMock(RequestParams):
    def __init__(self, **kvp):
        self.kvp = kvp

    def get_query_arguments(self) -> Mapping[str, str]:
        return dict(self.kvp)

    def get_query_argument(self, name: str, default: Optional[str] = UNDEFINED) -> Optional[str]:
        value = self.kvp.get(name, default)
        if value == UNDEFINED:
            raise ServiceBadRequestError(f'Missing query parameter "{name}"')
        return value


# test/webapi/test_app.py

import unittest


class AppSmokeTest(unittest.TestCase):

    def test_start_stop_service(self):
        pass
        # TODO: The following test code will cause timeouts in test/test_handlers.py - why?
        # service = new_service(args=['--port', '20001', '--update', '0'])
        # self.assertIsInstance(service, Service)
        # service.stop()
        # IOLoop.current().call_later(0.1, service.stop)
        # service.start()


# test/webapi/test_auth.py

import json
import os
import unittest

import requests

from xcube.webapi.auth import AuthMixin
from xcube.webapi.auth import assert_scopes
from xcube.webapi.auth import check_scopes
from xcube.webapi.errors import ServiceAuthError
from xcube.webapi.errors import ServiceConfigError


class ServiceContextMock:
    def __init__(self, config):
        self.config = config


class RequestMock:
    def __init__(self, headers):
        self.headers = headers


XCUBE_TEST_CLIENT_ID = os.environ.get('XCUBE_TEST_CLIENT_ID')
XCUBE_TEST_CLIENT_SECRET = os.environ.get('XCUBE_TEST_CLIENT_SECRET')


class AuthMixinIdTokenTest(unittest.TestCase):

    @unittest.skipUnless(
        XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET,
        'XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET must be set'
    )
    def test_ok(self):
        access_token = self._fetch_access_token()

        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/'
            )))
        auth_mixin.request = RequestMock(
            headers={'Authorization': f'Bearer {access_token}'}
        )

        id_token = auth_mixin.get_id_token()
        self.assertEqual('https://xcube-dev.eu.auth0.com/',
                         id_token.get('iss'))
        self.assertEqual('https://xcube-dev/api/',
                         id_token.get('aud'))
        self.assertEqual('w2NoDEryIpNRtAQVH1ToU6XTurE549FW',
                         id_token.get('azp'))
        self.assertEqual('client-credentials',
                         id_token.get('gty'))
        self.assertEqual('w2NoDEryIpNRtAQVH1ToU6XTurE549FW@clients',
                         id_token.get('sub'))
        self.assertEqual(['read:datasets'],
                         id_token.get('permissions'))
        exp = id_token.get('exp')
        iat = id_token.get('iat')
        self.assertIsInstance(exp, int)
        self.assertIsInstance(iat, int)
        self.assertEqual(86400, exp - iat)

    def _fetch_access_token(self):
        response = requests.post(
            'https://xcube-dev.eu.auth0.com/oauth/token',
            headers={'Content-Type': 'application/json'},
            json={
                "client_id": os.environ.get('XCUBE_TEST_CLIENT_ID'),
                "client_secret": os.environ.get('XCUBE_TEST_CLIENT_SECRET'),
                "audience": "https://xcube-dev/api/",
                "grant_type": "client_credentials"
            }
        )
        token_data = json.loads(response.content)
        if 'access_token' not in token_data:
            self.fail('failed to fetch access token for testing')
        access_token = token_data['access_token']
        return access_token

    def test_missing_auth_config(self):
        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config={})
        auth_mixin.request = RequestMock(
            headers={'Authorization': f'Bearer my_t0k3n'}
        )

        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_id_token(require_auth=True)

        self.assertEqual('HTTP 401: Invalid header (Received access token, '
                         'but this server doesn\'t support authentication.)',
                         f'{cm.exception}')

    def test_missing_access_token(self):
        auth_mixin = AuthMixin()
        auth_mixin.request = RequestMock(headers={})
        self.assertEqual(None, auth_mixin.get_id_token())

    @unittest.skipUnless(
        XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET,
        'XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET must be set'
    )
    def test_expired_access_token(self):
        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/'
            )))

        expired_token = \
            "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6" \
            "Ik5VSkdSVUV5UWpNeE16UTVRVGMzTnpRM05URkNRa1ky" \
            "TURGRE5qQkJSak5ETlRBeVFrWXdOUSJ9.eyJpc3MiOiJ" \
            "odHRwczovL3hjdWJlLWRldi5ldS5hdXRoMC5jb20vIiw" \
            "ic3ViIjoidzJOb0RFcnlJcE5SdEFRVkgxVG9VNlhUdXJ" \
            "FNTQ5RldAY2xpZW50cyIsImF1ZCI6Imh0dHBzOi8veGN" \
            "1YmUtZGV2L2FwaS8iLCJpYXQiOjE1Nzk0NTUxMDMsImV" \
            "4cCI6MTU3OTU0MTUwMywiYXpwIjoidzJOb0RFcnlJcE5" \
            "SdEFRVkgxVG9VNlhUdXJFNTQ5RlciLCJndHkiOiJjbGl" \
            "lbnQtY3JlZGVudGlhbHMiLCJwZXJtaXNzaW9ucyI6W11" \
            "9.NtGuVp0yY8foR_eFiEmH0eXvCx85cvb5b1cPWCKs6L" \
            "CNniUJJ3VnX5Fek08puW7Jidy-tj1UTzyG569dSHGDu3" \
            "10Mf7xpQ9gyfZCWcaohERxsv9MrxHziqfGVnxv051rOB" \
            "_c-fYyymrFnlsIqWeahcS7znvPoMovPO7E8MxVaIqxd_" \
            "S4zKVlH025F-bDvytuwXD-rFmYVElCg7u2uOZKqjpF3l" \
            "ZCWc50_F1jSGcEQZv4daQJY-3lfU6TnEQAuGWlOVRrCN" \
            "u05nlUBFPz6G82tB_nsP1uTa8uElOzoalVttXufLIeU0" \
            "FL8Sv-lC6wUJTZAFpykLNmpA-vhkSeTqMv4g"
        auth_mixin.request = RequestMock(
            headers={'Authorization': f'Bearer {expired_token}'})
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_id_token()
        self.assertEqual('HTTP 401: Token expired (Token is expired)',
                         f'{cm.exception}')


class AuthMixinAccessTokenTest(unittest.TestCase):

    def test_ok(self):
        auth_mixin = AuthMixin()
        auth_mixin.request = RequestMock(
            headers={'Authorization': 'Bearer my_t0k3n'}
        )
        self.assertEqual('my_t0k3n', auth_mixin.get_access_token())

    def test_ok_no_auth(self):
        auth_mixin = AuthMixin()
        auth_mixin.request = RequestMock(headers={})
        self.assertEqual(None, auth_mixin.get_access_token())

    def test_not_ok(self):
        auth_mixin = AuthMixin()

        auth_mixin.request = RequestMock(headers={})
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_access_token(require_auth=True)
        self.assertEqual('HTTP 401: Authorization header'
                         ' missing (Authorization header is expected)',
                         f'{cm.exception}')

        auth_mixin.request = RequestMock(
            headers={'Authorization': 'Beerer my_t0k3n'}
        )
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_access_token()
        self.assertEqual('HTTP 401: Invalid header'
                         ' (Authorization header must start with "Bearer")',
                         f'{cm.exception}')

        auth_mixin.request = RequestMock(
            headers={'Authorization': 'Bearer'}
        )
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_access_token()
        self.assertEqual('HTTP 401: Invalid header (Bearer token not found)',
                         f'{cm.exception}')

        auth_mixin.request = RequestMock(
            headers={'Authorization': 'Bearer my t0k3n'}
        )
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_access_token()
        self.assertEqual('HTTP 401: Invalid header'
                         ' (Authorization header must be Bearer token)',
                         f'{cm.exception}')


class AuthMixinAuthConfigTest(unittest.TestCase):

    def test_ok(self):
        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/'
            )))
        auth_config = auth_mixin.auth_config
        self.assertEqual('xcube-dev.eu.auth0.com',
                         auth_config.domain)
        self.assertEqual('https://xcube-dev/api/',
                         auth_config.audience)
        self.assertEqual('https://xcube-dev.eu.auth0.com/',
                         auth_config.issuer)
        self.assertEqual('https://xcube-dev.eu.auth0.com/'
                         '.well-known/jwks.json',
                         auth_config.well_known_jwks)
        self.assertEqual(['RS256'], auth_config.algorithms)

    def test_not_ok(self):
        # class ServiceContextMock:
        #     def __init__(self, config):
        #         self.config = config

        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
            )))

        with self.assertRaises(ServiceConfigError) as cm:
            # noinspection PyUnusedLocal
            auth_config = auth_mixin.auth_config
        self.assertEqual('HTTP 500: Missing key'
                         ' "Audience" in section "Authentication"',
                         f'{cm.exception}')

        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Audience='https://xcube-dev/api/'
            )))

        with self.assertRaises(ServiceConfigError) as cm:
            # noinspection PyUnusedLocal
            auth_config = auth_mixin.auth_config
        self.assertEqual('HTTP 500: Missing'
                         ' key "Domain" in section "Authentication"',
                         f'{cm.exception}')

        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/',
                Algorithms=[],
            )))

        with self.assertRaises(ServiceConfigError) as cm:
            # noinspection PyUnusedLocal
            auth_config = auth_mixin.auth_config
        self.assertEqual('HTTP 500: Value for key "Algorithms"'
                         ' in section "Authentication" must not be empty',
                         f'{cm.exception}')


class ScopesTest(unittest.TestCase):

    def test_check_scopes_ok(self):
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         None,
                         is_substitute=True)
        )
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test1.zarr'})
        )
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test?.zarr'})
        )
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test1.*'})
        )

    def test_check_scopes_fails(self):
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test1.zarr'},
                         None,
                         is_substitute=False)
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test1.zarr'},
                         set(),
                         is_substitute=True)
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test1.zarr'},
                         is_substitute=True)
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test2.zarr'},
                         {'read:dataset:test1.zarr'})
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test2.zarr'},
                         {'read:dataset:test1.zarr'},
                         is_substitute=True)
        )

    def test_assert_scopes_ok(self):
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test1.zarr'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:*'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test?.zarr'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test1.*'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test2.zarr',
                       'read:dataset:test3.zarr',
                       'read:dataset:test1.zarr'})

    def test_assert_scopes_fails(self):
        with self.assertRaises(ServiceAuthError) as cm:
            assert_scopes({'read:dataset:test1.zarr'},
                          None)
        self.assertEquals(
            'HTTP 401: Missing permission'
            ' (Missing permission read:dataset:test1.zarr)',
            f'{cm.exception}'
        )

        with self.assertRaises(ServiceAuthError) as cm:
            assert_scopes({'read:dataset:test1.zarr'},
                          set())
        self.assertEquals(
            'HTTP 401: Missing permission'
            ' (Missing permission read:dataset:test1.zarr)',
            f'{cm.exception}'
        )

        with self.assertRaises(ServiceAuthError) as cm:
            assert_scopes({'read:dataset:test1.zarr'},
                          {'read:dataset:test2.zarr'})
        self.assertEquals(
            'HTTP 401: Missing permission'
            ' (Missing permission read:dataset:test1.zarr)',
            f'{cm.exception}'
        )


# test/webapi/test_config.py

import os.path
import unittest

from xcube.util.jsonschema import JsonObjectSchema
from xcube.webapi.config import DatasetConfig
from xcube.webapi.config import ServiceConfig


class ServiceConfigTest(unittest.TestCase):
    def test_get_schema(self):
        schema = ServiceConfig.get_schema()
        self.assertIsInstance(schema, JsonObjectSchema)

    def test_from_dict(self):
        test_dir = os.path.dirname(__file__)
        service_config_dict = {
            "Authentication": {
                "Domain": "xcube - dev.eu.auth0.com",
                "Audience": "https://xcube-dev/api/",
            },
            "DatasetAttribution":
                [
                    "© by Brockmann Consult GmbH 2020",
                    "© by EU H2020 CyanoAlert project",
                ],
            "DatasetChunkCacheSize": "100M",
            "Datasets": [
                {
                    # Will only appear for unauthorized clients
                    "Identifier": "local",
                    "Title": "Local OLCI L2C cube for region SNS",
                    "BoundingBox": [0.0, 50, 5.0, 52.5],
                    "FileSystem": "file",
                    "Path": "cube-1-250-250.zarr",
                    "Style": "default",
                    "TimeSeriesDataset": "local_ts",
                    "Augmentation": {
                        "Path": "compute_extra_vars.py",
                        "Function": "compute_variables",
                        "InputParameters": {
                            "factor_chl": 0.2,
                            "factor_tsm": 0.7,
                        },
                    },
                    "PlaceGroups": [
                        {"PlaceGroupRef": "inside-cube"},
                        {"PlaceGroupRef": "outside-cube"},
                    ],
                    "AccessControl": {
                        "IsSubstitute": True,
                    },
                }
            ],
            "DataStores": [
                {
                    "Identifier": "local",
                    "StoreId": "file",
                    "StoreParams": {
                        "root": f"{test_dir}/../../examples/serve/demo",
                    },
                    "Datasets": [
                        {
                            "Path": "*.zarr",
                            "Style": "default"
                        }
                    ]
                },
            ]
        }
        service_config = ServiceConfig.from_dict(service_config_dict)
        self.assertIsInstance(service_config, ServiceConfig)
        self.assertTrue(hasattr(service_config, 'Datasets'))
        self.assertIsInstance(service_config.Datasets, list)
        self.assertEqual(1, len(service_config.Datasets))
        self.assertIsInstance(service_config.Datasets[0], DatasetConfig)
        self.assertTrue(hasattr(service_config.Datasets[0], 'Identifier'))
        self.assertEqual("local", service_config.Datasets[0].Identifier)


# test/webapi/test_context.py

import os.path
import unittest

import xarray as xr

from test.webapi.helpers import new_test_service_context
from xcube.core.mldataset import MultiLevelDataset
from xcube.webapi.context import normalize_prefix
from xcube.webapi.errors import ServiceResourceNotFoundError


class ServiceContextTest(unittest.TestCase):
    def test_get_dataset_and_variable(self):
        ctx = new_test_service_context()

        ds = ctx.get_dataset('demo')
        self.assertIsInstance(ds, xr.Dataset)

        ml_ds = ctx.get_ml_dataset('demo')
        self.assertIsInstance(ml_ds, MultiLevelDataset)
        self.assertIs(3, ml_ds.num_levels)
        self.assertIs(ds, ml_ds.get_dataset(0))

        for var_name in ('conc_chl', 'conc_tsm'):
            for z in range(ml_ds.num_levels):
                conc_chl_z = ctx.get_variable_for_z('demo', var_name, z)
                self.assertIsInstance(conc_chl_z, xr.DataArray)
            with self.assertRaises(ServiceResourceNotFoundError) as cm:
                ctx.get_variable_for_z('demo', var_name, 3)
            self.assertEqual(404, cm.exception.status_code)
            self.assertEqual(f'Variable "{var_name}" has no z-index 3 in dataset "demo"', cm.exception.reason)

        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            ctx.get_variable_for_z('demo', 'conc_ys', 0)
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Variable "conc_ys" not found in dataset "demo"', cm.exception.reason)

        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            ctx.get_dataset('demox')
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Dataset "demox" not found', cm.exception.reason)

        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            ctx.get_dataset('demo', expected_var_names=['conc_ys'])
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Variable "conc_ys" not found in dataset "demo"', cm.exception.reason)

    def test_get_dataset_with_augmentation(self):
        ctx = new_test_service_context(config_file_name='config-aug.yml')

        ds = ctx.get_dataset('demo-aug')
        self.assertIsInstance(ds, xr.Dataset)

        ml_ds = ctx.get_ml_dataset('demo-aug')
        self.assertIsInstance(ml_ds, MultiLevelDataset)
        self.assertIs(3, ml_ds.num_levels)
        self.assertIs(ds, ml_ds.get_dataset(0))

        for var_name in ('conc_chl', 'conc_tsm', 'chl_tsm_sum', 'chl_category'):
            for z in range(ml_ds.num_levels):
                conc_chl_z = ctx.get_variable_for_z('demo-aug', var_name, z)
                self.assertIsInstance(conc_chl_z, xr.DataArray)

    def test_get_dataset_configs_from_stores(self):
        ctx = new_test_service_context(config_file_name='config-datastores.yml')

        dataset_configs_from_stores = ctx.get_dataset_configs_from_stores()
        self.assertIsNotNone(dataset_configs_from_stores)
        self.assertEqual(3, len(dataset_configs_from_stores))
        ids = [config['Identifier'] for config in dataset_configs_from_stores]
        self.assertIn('test~cube-1-250-250.zarr', ids)
        self.assertIn('test~cube-5-100-200.zarr', ids)
        self.assertIn('test~cube-1-250-250.levels', ids)

    def test_config_and_dataset_cache(self):
        ctx = new_test_service_context()
        self.assertNotIn('demo', ctx.dataset_cache)

        ctx.get_dataset('demo')
        self.assertIn('demo', ctx.dataset_cache)

        ctx.config = dict(Datasets=[
            dict(Identifier='demo',
                 Path="../../../../examples/serve/demo/cube.nc"),
            dict(Identifier='demo2',
                 Path="../../../../examples/serve/demo/cube.nc"),
        ])
        self.assertNotIn('demo', ctx.dataset_cache)
        self.assertNotIn('demo2', ctx.dataset_cache)

        ctx.get_dataset('demo2')
        self.assertNotIn('demo', ctx.dataset_cache)
        self.assertIn('demo2', ctx.dataset_cache)

        ctx.config = dict(Datasets=[
            dict(Identifier='demo2',
                 Path="../../../../examples/serve/demo/cube.nc"),
        ])
        self.assertNotIn('demo', ctx.dataset_cache)
        self.assertNotIn('demo2', ctx.dataset_cache)

    def test_get_s3_bucket_mapping(self):
        ctx = new_test_service_context()
        bucket_mapping = ctx.get_s3_bucket_mapping()
        self.assertEqual(['demo'],
                         list(bucket_mapping.keys()))
        path = bucket_mapping['demo']
        self.assertTrue(os.path.isabs(path))
        self.assertTrue(path.replace('\\', '/').endswith('examples/serve/demo/cube-1-250-250.zarr'))

    def test_get_color_mapping(self):
        ctx = new_test_service_context()
        cm = ctx.get_color_mapping('demo', 'conc_chl')
        self.assertEqual(('plasma', (0., 24.)), cm)
        cm = ctx.get_color_mapping('demo', 'conc_tsm')
        self.assertEqual(('PuBuGn', (0., 100.)), cm)
        cm = ctx.get_color_mapping('demo', 'kd489')
        self.assertEqual(('jet', (0., 6.)), cm)
        with self.assertRaises(ServiceResourceNotFoundError):
            ctx.get_color_mapping('demo', '_')

    def test_get_rgb_color_mapping(self):
        ctx = new_test_service_context()
        rgb_cm = ctx.get_rgb_color_mapping('demo')
        self.assertEqual(([None, None, None], [(0.0, 1.0), (0.0, 1.0), (0.0, 1.0)]), rgb_cm)
        rgb_cm = ctx.get_rgb_color_mapping('demo', norm_range=(1.0, 2.5))
        self.assertEqual(([None, None, None], [(1.0, 2.5), (1.0, 2.5), (1.0, 2.5)]), rgb_cm)
        ctx = new_test_service_context('config-rgb.yml')
        rgb_cm = ctx.get_rgb_color_mapping('demo-rgb')
        self.assertEqual((['conc_chl', 'conc_tsm', 'kd489'], [(0.0, 24.0), (0.0, 100.0), (0.0, 6.0)]), rgb_cm)

    def test_get_style(self):
        ctx = new_test_service_context()
        style = ctx.get_style('demo-1w')
        self.assertEqual({'Identifier': 'default',
                          'ColorMappings': {'conc_chl': {'ColorBar': 'plasma',
                                                         'ValueRange': [0.0, 24.0]},
                                            'conc_tsm': {'ColorBar': 'PuBuGn',
                                                         'ValueRange': [0.0, 100.0]},
                                            'kd489': {'ColorBar': 'jet', 'ValueRange': [0.0, 6.0]}}},
                         style)

    def test_get_global_place_groups(self):
        ctx = new_test_service_context()
        place_groups = ctx.get_global_place_groups("http://localhost:9090", load_features=False)
        self.assertIsInstance(place_groups, list)
        self.assertEqual(2, len(place_groups))
        for place_group in place_groups:
            self.assertIn("type", place_group)
            self.assertIn("id", place_group)
            self.assertIn("title", place_group)
            self.assertIn("propertyMapping", place_group)
            self.assertIn("sourcePaths", place_group)
            self.assertIn("sourceEncoding", place_group)
            self.assertIn("features", place_group)
        place_group = place_groups[0]
        self.assertEqual('inside-cube', place_group['id'])
        self.assertEqual('Points inside the cube', place_group['title'])
        self.assertEqual(None, place_group['features'])
        place_group = place_groups[1]
        self.assertEqual('outside-cube', place_group['id'])
        self.assertEqual('Points outside the cube', place_group['title'])
        self.assertEqual(None, place_group['features'])

        place_groups = ctx.get_global_place_groups("http://localhost:9090", load_features=True)
        self.assertIsInstance(place_groups, list)
        self.assertEqual(2, len(place_groups))
        for place_group in place_groups:
            self.assertIn("type", place_group)
            self.assertIn("id", place_group)
            self.assertIn("title", place_group)
            self.assertIn("propertyMapping", place_group)
            self.assertIn("sourcePaths", place_group)
            self.assertIn("sourceEncoding", place_group)
            self.assertIn("features", place_group)
        place_group = place_groups[0]
        self.assertEqual('inside-cube', place_group['id'])
        self.assertEqual('Points inside the cube', place_group['title'])
        self.assertIsNotNone(place_group['features'])
        place_group = place_groups[1]
        self.assertEqual('outside-cube', place_group['id'])
        self.assertEqual('Points outside the cube', place_group['title'])
        self.assertIsNotNone(place_group['features'])

    def test_get_global_place_group(self):
        ctx = new_test_service_context()
        place_group = ctx.get_global_place_group("inside-cube", "http://localhost:9090", load_features=True)
        self.assertIsInstance(place_group, dict)
        self.assertIn("type", place_group)
        self.assertEqual("FeatureCollection", place_group["type"])
        self.assertIn("features", place_group)
        self.assertIsInstance(place_group["features"], list)
        self.assertEqual(3, len(place_group["features"]))
        self.assertIs(place_group, ctx.get_global_place_group("inside-cube", "http://localhost:9090"))
        self.assertIsNot(place_group, ctx.get_global_place_group("outside-cube", "http://localhost:9090"))

        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            ctx.get_global_place_group("bibo", "http://localhost:9090")
        self.assertEqual('HTTP 404: Place group "bibo" not found', f"{cm.exception}")

    def test_get_other_store_params_than_root(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'two',
                          'Title': 'Test 2',
                          'FileSystem': 's3',
                          'Anonymous': False,
                          'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                          'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                          'Region': 'eu-central-1'}
        store_params = ctx._get_other_store_params_than_root(dataset_config)
        expected_dict = \
            {'storage_options':
                 {'anon': False,
                  'client_kwargs':
                      {'endpoint_url': 'https://s3.eu-central-1.amazonaws.com',
                       'region_name': 'eu-central-1'}
                  }
             }
        self.assertIsNotNone(store_params)
        self.assertEqual(expected_dict, store_params)


class NormalizePrefixTest(unittest.TestCase):
    def test_empty(self):
        self.assertEqual('', normalize_prefix(None))
        self.assertEqual('', normalize_prefix(''))
        self.assertEqual('', normalize_prefix('/'))
        self.assertEqual('', normalize_prefix('///'))

    def test_normalizes_slashes(self):
        self.assertEqual('/xcube/api/v1', normalize_prefix('/xcube/api/v1'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('xcube/api/v1'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('xcube/api/v1/'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('/xcube/api/v1/'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('/xcube/api/v1///'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('////xcube/api/v1'))

    def test_interpolates_vars(self):
        from xcube.version import version
        self.assertEqual(f'/{version}',
                         normalize_prefix('${version}'))
        self.assertEqual(f'/xcube',
                         normalize_prefix('/${name}'))
        self.assertEqual(f'/xcube/v{version}',
                         normalize_prefix('/${name}/v${version}'))


class MaybeAssignStoreInstanceIdsTest(unittest.TestCase):

    def test_find_common_store(self):
        ctx = new_test_service_context()
        dataset_configs = [
            {
                'Identifier': 'z_0',
                'FileSystem': 'file',
                'Path': '/one/path/abc.zarr'
            },
            {
                'Identifier': 'z_1',
                'FileSystem': 'file',
                'Path': '/one/path/def.zarr'
            },
            {
                'Identifier': 'z_4',
                'FileSystem': 's3',
                'Path': '/one/path/mno.zarr'
            },
            {
                'Identifier': 'z_2',
                'FileSystem': 'file',
                'Path': '/another/path/ghi.zarr'
            },
            {
                'Identifier': 'z_3',
                'FileSystem': 'file',
                'Path': '/one/more/path/jkl.zarr'
            },
            {
                'Identifier': 'z_5',
                'FileSystem': 's3',
                'Path': '/one/path/pqr.zarr'
            },
            {
                'Identifier': 'z_6',
                'FileSystem': 'file',
                'Path': '/one/path/stu.zarr'
            },
            {
                'Identifier': 'z_7',
                'FileSystem': 'file',
                'Path': '/one/more/path/vwx.zarr'
            },
        ]
        ctx.config['Datasets'] = dataset_configs
        adjusted_dataset_configs = ctx.get_dataset_configs()

        expected_dataset_configs = [
            {
                'Identifier': 'z_0',
                'FileSystem': 'file',
                'Path': 'path/abc.zarr',
                'StoreInstanceId': 'file_2'
            },
            {
                'Identifier': 'z_1',
                'FileSystem': 'file',
                'Path': 'path/def.zarr',
                'StoreInstanceId': 'file_2'
            },
            {
                'Identifier': 'z_4',
                'FileSystem': 's3',
                'Path': 'mno.zarr',
                'StoreInstanceId': 's3_1'
            },
            {
                'Identifier': 'z_2',
                'FileSystem': 'file',
                'Path': 'ghi.zarr',
                'StoreInstanceId': 'file_1'
            },
            {
                'Identifier': 'z_3',
                'FileSystem': 'file',
                'Path': 'more/path/jkl.zarr',
                'StoreInstanceId': 'file_2'
            },
            {
                'Identifier': 'z_5',
                'FileSystem': 's3',
                'Path': 'pqr.zarr',
                'StoreInstanceId': 's3_1'
            },
            {
                'Identifier': 'z_6',
                'FileSystem': 'file',
                'Path': 'path/stu.zarr',
                'StoreInstanceId': 'file_2'
            },
            {
                'Identifier': 'z_7',
                'FileSystem': 'file',
                'Path': 'more/path/vwx.zarr',
                'StoreInstanceId': 'file_2'
            },
        ]
        self.assertEqual(expected_dataset_configs, adjusted_dataset_configs)

    def test_with_instance_id(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'zero',
                          'Title': 'Test 0',
                          'FileSystem': 'file',
                          'StoreInstanceId': 'some_id'}
        dataset_config_copy = dataset_config.copy()

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(dataset_config_copy, dataset_config)

    def test_file(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'one',
                          'Title': 'Test 1',
                          'FileSystem': 'file',
                          'Path': 'cube-1-250-250.zarr'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Path',
                         'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('one',
                         dataset_config['Identifier'])
        self.assertEqual('Test 1', dataset_config['Title'])
        self.assertEqual('file', dataset_config['FileSystem'])
        self.assertEqual('cube-1-250-250.zarr', dataset_config["Path"])
        self.assertEqual('file_1', dataset_config['StoreInstanceId'])

    def test_local(self):
        # this test tests backwards compatibility.
        # TODO please remove when support for file systems 'local' and 'obs'
        # has ended
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'one',
                          'Title': 'Test 1',
                          'FileSystem': 'local',
                          'Path': 'cube-1-250-250.zarr'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Path',
                         'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('one',
                         dataset_config['Identifier'])
        self.assertEqual('Test 1', dataset_config['Title'])
        self.assertEqual('local', dataset_config['FileSystem'])
        self.assertEqual('cube-1-250-250.zarr', dataset_config["Path"])
        self.assertEqual('file_1', dataset_config['StoreInstanceId'])

    def test_s3(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'two',
                          'Title': 'Test 2',
                          'FileSystem': 's3',
                          'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                          'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                          'Region': 'eu-central-1'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Endpoint',
                          'Path', 'Region', 'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('two', dataset_config['Identifier'])
        self.assertEqual('Test 2', dataset_config['Title'])
        self.assertEqual('s3', dataset_config['FileSystem'])
        self.assertEqual('https://s3.eu-central-1.amazonaws.com',
                         dataset_config['Endpoint'])
        self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path'])
        self.assertEqual('eu-central-1', dataset_config['Region'])
        self.assertEqual('s3_1', dataset_config['StoreInstanceId'])

    def test_obs(self):
        # this test tests backwards compatibility.
        # TODO please remove when support for file systems 'local' and 'obs'
        # has ended
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'two',
                          'Title': 'Test 2',
                          'FileSystem': 'obs',
                          'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                          'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                          'Region': 'eu-central-1'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Endpoint',
                          'Path', 'Region', 'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('two', dataset_config['Identifier'])
        self.assertEqual('Test 2', dataset_config['Title'])
        self.assertEqual('obs', dataset_config['FileSystem'])
        self.assertEqual('https://s3.eu-central-1.amazonaws.com',
                         dataset_config['Endpoint'])
        self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path'])
        self.assertEqual('eu-central-1', dataset_config['Region'])
        self.assertEqual('s3_1', dataset_config['StoreInstanceId'])

    def test_memory(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'three',
                          'Title': 'Test 3',
                          'FileSystem': 'memory'}
        dataset_config_copy = dataset_config.copy()

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(dataset_config_copy, dataset_config)

    def test_missing_file_system(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'five',
                          'Title': 'Test 5',
                          'Path': 'cube-1-250-250.zarr'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'Path', 'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('five', dataset_config['Identifier'])
        self.assertEqual('Test 5', dataset_config['Title'])
        self.assertEqual('cube-1-250-250.zarr', dataset_config['Path'])
        self.assertEqual('file_1', dataset_config['StoreInstanceId'])

    def test_invalid_file_system(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'five',
                          'Title': 'Test 5a',
                          'FileSystem': 'invalid',
                          'Path': 'cube-1-250-250.zarr'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Path'],
                         list(dataset_config.keys()))
        self.assertEqual('five', dataset_config['Identifier'])
        self.assertEqual('Test 5a', dataset_config['Title'])
        self.assertEqual('invalid', dataset_config['FileSystem'])
        self.assertEqual('cube-1-250-250.zarr', dataset_config['Path'])

    def test_local_store_already_existing(self):
        ctx = new_test_service_context()
        dataset_config_1 = {'Identifier': 'six',
                            'Title': 'Test 6',
                            'FileSystem': 'file',
                            'Path': 'cube-1-250-250.zarr'}
        dataset_config_2 = {'Identifier': 'six_a',
                            'Title': 'Test 6 a',
                            'FileSystem': 'file',
                            'Path': 'cube-5-100-200.zarr'}

        ctx.config['Datasets'] = [dataset_config_1, dataset_config_2]
        dataset_configs = ctx.get_dataset_configs()

        self.assertEqual(dataset_configs[0]['StoreInstanceId'],
                         dataset_configs[1]['StoreInstanceId'])

    def test_s3_store_already_existing(self):
        ctx = new_test_service_context()
        dataset_config_1 = {'Identifier': 'seven',
                            'Title': 'Test 7',
                            'FileSystem': 's3',
                            'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                            'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                            'Region': 'eu-central-1'}

        dataset_config_2 = {'Identifier': 'seven_a',
                            'Title': 'Test 7 a',
                            'FileSystem': 's3',
                            'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                            'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-3.zarr',
                            'Region': 'eu-central-1'}

        ctx.config['Datasets'] = [dataset_config_1, dataset_config_2]
        dataset_configs = ctx.get_dataset_configs()

        self.assertEqual(dataset_configs[0]['StoreInstanceId'],
                         dataset_configs[1]['StoreInstanceId'])


# test/webapi/test_errors.py

import unittest

from tornado.web import HTTPError

from xcube.webapi.errors import ServiceError, ServiceConfigError, ServiceBadRequestError, ServiceResourceNotFoundError


class ErrorsTest(unittest.TestCase):
    def test_same_base_type(self):

        self.assertIsInstance(ServiceError(''), HTTPError)
        self.assertEqual(500, ServiceError('').status_code)
        self.assertEqual(503, ServiceError('', status_code=503).status_code)

        self.assertIsInstance(ServiceConfigError(''), ServiceError)
        self.assertEqual(500, ServiceConfigError('').status_code)

        self.assertIsInstance(ServiceBadRequestError(''), ServiceError)
        self.assertEqual(400, ServiceBadRequestError('').status_code)

        self.assertIsInstance(ServiceResourceNotFoundError(''), ServiceError)
        self.assertEqual(404, ServiceResourceNotFoundError('').status_code)


# test/webapi/test_handlers.py

from tornado.testing import AsyncHTTPTestCase

from test.webapi.helpers import new_test_service_context
from xcube.webapi.app import new_application

CTX = new_test_service_context()

# Preload datasets, so we don't run into timeout during testing

_ds = CTX.get_dataset("demo")
for name, var in _ds.coords.items():
    values = var.values

_ds = CTX.get_dataset("demo-1w")
for name, var in _ds.coords.items():
    values = var.values


# For usage of the tornado.testing.AsyncHTTPTestCase see http://www.tornadoweb.org/en/stable/testing.html


class HandlersTest(AsyncHTTPTestCase):

    def get_app(self):
        application = new_application()
        application.service_context = CTX
        return application

    def assertResponseOK(self, response):
        self.assertEqual(200, response.code, response.reason)
        self.assertEqual("OK", response.reason)

    def assertBadRequestResponse(self, response, expected_reason="Bad Request"):
        self.assertEqual(400, response.code)
        self.assertEqual(expected_reason, response.reason)

    def assertResourceNotFoundResponse(self, response, expected_reason="Not Found"):
        self.assertEqual(404, response.code)
        self.assertEqual(expected_reason, response.reason)

    def test_fetch_base(self):
        response = self.fetch(self.prefix + '/')
        self.assertResponseOK(response)

    def test_fetch_wmts_kvp_capabilities(self):
        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?SERVICE=WMTS'
                                            '&VERSION=1.0.0'
                                            '&REQUEST=GetCapabilities')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?service=WMTS'
                                            '&version=1.0.0'
                                            '&request=GetCapabilities')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetCapabilities')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?VERSION=1.0.0&REQUEST=GetCapabilities')
        self.assertBadRequestResponse(response, 'Missing query parameter "service"')

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?SERVICE=WMS'
                                            'VERSION=1.0.0'
                                            '&REQUEST=GetCapabilities')
        self.assertBadRequestResponse(response, 'Value for "service" parameter must be "WMTS"')

    def test_fetch_wmts_kvp_tile(self):
        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetTile'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=TileGrid_2000_1000'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0')
        self.assertResponseOK(response)

        # issue #132 by Dirk
        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetTile'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=TileGrid_2000_1000'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0'
                                            '&Time=2017-01-25T09%3A35%3A50')
        self.assertResponseOK(response)

        # issue #132 by Dirk
        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetTile'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=TileGrid_2000_1000'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0'
                                            '&Time=2017-01-25T09%3A35%3A50%2F2017-01-25T10%3A20%3A15')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetTile'
                                            '&Format=image/jpg'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=TileGrid_2000_1000'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0')
        self.assertBadRequestResponse(response, 'Value for "format" parameter must be "image/png"')

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.1.0'
                                            '&Request=GetTile'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=TileGrid_2000_1000'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0')
        self.assertBadRequestResponse(response, 'Value for "version" parameter must be "1.0.0"')

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Request=GetTile'
                                            '&Version=1.0.0'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=conc_chl'
                                            '&TileMatrixSet=TileGrid_2000_1000'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0')
        self.assertBadRequestResponse(response, 'Value for "layer" parameter must be "<dataset>.<variable>"')

    def test_fetch_wmts_capabilities(self):
        response = self.fetch(self.prefix + '/wmts/1.0.0/WMTSCapabilities.xml')
        self.assertResponseOK(response)

    def test_fetch_wmts_tile(self):
        response = self.fetch(self.prefix + '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png')
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_with_params(self):
        response = self.fetch(self.prefix + '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png?time=current&cbar=jet')
        self.assertResponseOK(response)

    def test_fetch_datasets(self):
        response = self.fetch(self.prefix + '/datasets')
        self.assertResponseOK(response)

    def test_fetch_datasets_details(self):
        response = self.fetch(self.prefix + '/datasets?details=1')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/datasets?details=1&tiles=ol')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/datasets?details=1&point=2.8,51.0')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/datasets?details=1&point=2,8a,51.0')
        self.assertBadRequestResponse(response,
                                      "Parameter 'point' parameter must be a point using format"
                                      " '<lon>,<lat>', but was '2,8a,51.0'")

    def test_fetch_dataset(self):
        response = self.fetch(self.prefix + '/datasets/demo')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/datasets/demo?tiles=ol4')
        self.assertResponseOK(response)

    def test_fetch_dataset_coords(self):
        response = self.fetch(self.prefix + '/datasets/demo/coords/time')
        self.assertResponseOK(response)

    def test_fetch_list_s3bucket(self):
        response = self.fetch(self.prefix + '/s3bucket')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket?delimiter=%2F')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket?delimiter=%2F&prefix=demo%2F')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket?delimiter=%2F&list-type=2')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket?delimiter=%2F&prefix=demo%2F&list-type=2')
        self.assertResponseOK(response)

    def test_fetch_head_s3bucket_object(self):
        self._assert_fetch_head_s3bucket_object(method='HEAD')

    def test_fetch_get_s3bucket_object(self):
        self._assert_fetch_head_s3bucket_object(method='GET')

    def _assert_fetch_head_s3bucket_object(self, method):
        response = self.fetch(self.prefix + '/s3bucket/demo', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/.zattrs', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/.zgroup', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/.zarray', method=method)
        self.assertResourceNotFoundResponse(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/time/.zattrs', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/time/.zarray', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/time/.zgroup', method=method)
        self.assertResourceNotFoundResponse(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/time/0', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/.zattrs', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/.zarray', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/.zgroup', method=method)
        self.assertResourceNotFoundResponse(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/3.2.4', method=method)
        self.assertResponseOK(response)

    def test_fetch_dataset_tile(self):
        response = self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tiles/0/0/0.png')
        self.assertResponseOK(response)

    def test_fetch_dataset_rgb_tile(self):
        response = self.fetch(self.prefix + '/datasets/demo/vars/rgb/tiles/0/0/0.png?'
                                            'r=kd489&g=conc_chl&b=conc_tsm&time=current')
        self.assertResponseOK(response)

    def test_fetch_dataset_tile_with_params(self):
        response = self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tiles/0/0/0.png?'
                                            'time=current&cbar=jet&debug=1')
        self.assertResponseOK(response)

    def test_fetch_dataset_tile_grid_ol4_json(self):
        response = self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tilegrid?tiles=ol4')
        self.assertResponseOK(response)

    def test_fetch_dataset_tile_grid_cesium_json(self):
        self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tilegrid?tiles=cesium')

    def test_fetch_color_bars_json(self):
        response = self.fetch(self.prefix + '/colorbars')
        self.assertResponseOK(response)

    def test_fetch_color_bars_html(self):
        response = self.fetch(self.prefix + '/colorbars.html')
        self.assertResponseOK(response)

    def test_fetch_all_places(self):
        response = self.fetch(self.prefix + '/places')
        self.assertResponseOK(response)

    def test_fetch_places(self):
        response = self.fetch(self.prefix + '/places/all')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/places/all?bbox=10,10,20,20')
        self.assertResponseOK(response)

    def test_fetch_dataset_places(self):
        response = self.fetch(self.prefix + '/datasets/demo/places')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/places/all/demo')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/places/inside-cube/demo')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/places/bibo/demo')
        self.assertResourceNotFoundResponse(response, 'Place group "bibo" not found')
        response = self.fetch(self.prefix + '/places/inside-cube/bibo')
        self.assertResourceNotFoundResponse(response, 'Dataset "bibo" not found')

    def test_fetch_timeseries_invalid_body(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='')
        self.assertBadRequestResponse(response, 'Invalid or missing GeoJSON object in request body')

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type":"Point"}')
        self.assertBadRequestResponse(response, 'GeoJSON object expected')

    def test_fetch_timeseries_geometry(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "Point", "coordinates": [1, 51]}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type":"Polygon", "coordinates": [[[1, 51], [2, 51], [2, 52], [1, 51]]]}')
        self.assertResponseOK(response)

    def test_fetch_timeseries_geometry_collection(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "GeometryCollection", "geometries": null}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "GeometryCollection", "geometries": []}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "GeometryCollection", "geometries": '
                                   '[{"type": "Point", "coordinates": [1, 51]}]}')
        self.assertResponseOK(response)

    def test_fetch_timeseries_feature(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "Feature", '
                                   ' "properties": {}, '
                                   ' "geometry": {"type": "Point", "coordinates": [1, 51]}'
                                   '}')
        self.assertResponseOK(response)

    def test_fetch_timeseries_feature_collection(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "FeatureCollection", "features": null}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "FeatureCollection", "features": []}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "FeatureCollection", "features": ['
                                   '  {"type": "Feature", "properties": {}, '
                                   '   "geometry": {"type": "Point", "coordinates": [1, 51]}}'
                                   ']}')
        self.assertResponseOK(response)

    def test_fetch_ts_legacy_info(self):
        response = self.fetch(self.prefix + '/ts')
        self.assertResponseOK(response)

    def test_fetch_ts_legacy_point(self):
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point')
        self.assertBadRequestResponse(response, 'Missing query parameter "lon"')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point?lon=2.1')
        self.assertBadRequestResponse(response, 'Missing query parameter "lat"')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point?lon=2.1&lat=51.1&maxValids=-5')
        self.assertBadRequestResponse(response, 'If given, query parameter "maxValids" must be -1 or positive')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point?lon=120.5&lat=-12.4')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point?lon=2.1&lat=51.1')
        self.assertResponseOK(response)

    def test_fetch_ts_legacy_geometry(self):
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometry', method="POST",
                              body='')
        self.assertBadRequestResponse(response, 'Invalid or missing GeoJSON geometry in request body')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometry', method="POST",
                              body='{"type":"Point"}')
        self.assertBadRequestResponse(response, 'Invalid GeoJSON geometry')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometry', method="POST",
                              body='{"type": "Point", "coordinates": [1, 51]}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometry', method="POST",
                              body='{"type":"Polygon", "coordinates": [[[1, 51], [2, 51], [2, 52], [1, 51]]]}')

        self.assertResponseOK(response)

    def test_fetch_ts_legacy_geometries(self):
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='')
        self.assertBadRequestResponse(response, 'Invalid or missing GeoJSON geometry collection in request body')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='{"type":"Point"}')
        self.assertBadRequestResponse(response, 'Invalid GeoJSON geometry collection')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='{"type": "GeometryCollection", "geometries": null}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='{"type": "GeometryCollection", "geometries": []}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='{"type": "GeometryCollection", "geometries": '
                                   '[{"type": "Point", "coordinates": [1, 51]}]}')
        self.assertResponseOK(response)

    def test_fetch_ts_legacy_features(self):
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='')
        self.assertBadRequestResponse(response, 'Invalid or missing GeoJSON feature collection in request body')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='{"type":"Point"}')
        self.assertBadRequestResponse(response, 'Invalid GeoJSON feature collection')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='{"type": "FeatureCollection", "features": null}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='{"type": "FeatureCollection", "features": []}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='{"type": "FeatureCollection", "features": ['
                                   '  {"type": "Feature", "properties": {}, '
                                   '   "geometry": {"type": "Point", "coordinates": [1, 51]}}'
                                   ']}')
        self.assertResponseOK(response)

    @property
    def prefix(self):
        return ''


# test/webapi/test_reqparams.py

import unittest

from test.webapi.helpers import RequestParamsMock
from xcube.webapi.errors import ServiceBadRequestError
from xcube.webapi.reqparams import RequestParams


class RequestParamsTest(unittest.TestCase):
    def test_to_int(self):
        self.assertEqual(2123, RequestParams.to_int('x', '2123'))
        with self.assertRaises(ServiceBadRequestError):
            RequestParams.to_int('x', None)
        with self.assertRaises(ServiceBadRequestError):
            RequestParams.to_int('x', 'bibo')

    def test_to_float(self):
        self.assertEqual(-0.2, RequestParams.to_float('x', '-0.2'))
        with self.assertRaises(ServiceBadRequestError):
            RequestParams.to_float('x', None)
        with self.assertRaises(ServiceBadRequestError):
            RequestParams.to_float('x', 'bibo')

    def test_get_query_argument(self):
        rp = RequestParamsMock()
        self.assertEqual('bert', rp.get_query_argument('s', 'bert'))
        self.assertEqual(234, rp.get_query_argument_int('i', 234))
        self.assertEqual(0.2, rp.get_query_argument_float('f', 0.2))
        self.assertEqual((-10.4, 51.8), rp.get_query_argument_point('p', (-10.4, 51.8)))

        rp = RequestParamsMock(s='bibo', i='465', f='0.1', p='-10.4, 51.8')
        self.assertEqual('bibo', rp.get_query_argument('s', None))
        self.assertEqual(465, rp.get_query_argument_int('i', None))
        self.assertEqual(465., rp.get_query_argument_float('i', None))
        self.assertEqual(0.1, rp.get_query_argument_float('f', None))
        self.assertEqual((-10.4, 51.8), rp.get_query_argument_point('p', None))


# test/webapi/test_s3buckethandlers.py

import unittest
import urllib.request

import numpy as np
import xarray as xr

from xcube.core.dsio import open_cube

SKIP_HELP = ('Skipped, because server is not running:'
             ' $ xcube serve --verbose -c examples/serve/demo/config.yml')
SERVER_URL = 'http://localhost:8080'
ENDPOINT_URL = SERVER_URL + '/s3bucket'


def is_server_running() -> bool:
    # noinspection PyBroadException
    try:
        with urllib.request.urlopen(SERVER_URL, timeout=2.0) as response:
            response.read()
    except Exception:
        return False
    return 200 <= response.code < 400


XCUBE_SERVER_IS_RUNNING = is_server_running()


class S3BucketHandlersTest(unittest.TestCase):

    @unittest.skipUnless(XCUBE_SERVER_IS_RUNNING, SKIP_HELP)
    def test_open_cube_from_xube_server_rel_path(self):
        ds = open_cube('s3bucket/local',
                       format_name='zarr',
                       s3_kwargs={
                           'anon': True
                       },
                       s3_client_kwargs=dict(endpoint_url=SERVER_URL))
        self.assertCubeOk(ds)

    @unittest.skipUnless(XCUBE_SERVER_IS_RUNNING, SKIP_HELP)
    def test_open_cube_from_xube_server_abs_path(self):
        ds = open_cube('http://localhost:8080/s3bucket/local',
                       format_name='zarr',
                       s3_kwargs={
                           'anon': True
                       })
        self.assertCubeOk(ds)

    def assertCubeOk(self, ds):
        self.assertIsInstance(ds, xr.Dataset)
        self.assertEqual((5, 1000, 2000), ds.conc_chl.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.conc_chl.dims)
        conc_chl_values = ds.conc_chl.values
        self.assertEqual((5, 1000, 2000), conc_chl_values.shape)
        self.assertAlmostEqual(0.00005656,
                               float(np.nanmin(conc_chl_values)),
                               delta=1e-6)
        self.assertAlmostEqual(22.4421215,
                               float(np.nanmax(conc_chl_values)),
                               delta=1e-6)


# test/webapi/test_s3util.py

import os.path
import os.path
import unittest
from abc import abstractmethod, ABCMeta
from typing import Dict

from xcube.core.dsio import rimraf
from xcube.core.dsio import write_cube
from xcube.core.new import new_cube
from xcube.webapi.s3util import list_bucket_result_to_xml, list_s3_bucket_keys, list_s3_bucket_v2, list_s3_bucket_v1

S3_BUCKET = os.path.join(os.path.dirname(__file__), "s3-bucket")
TEST_CUBE_1 = os.path.join(S3_BUCKET, "test-1.zarr")
TEST_CUBE_2 = os.path.join(S3_BUCKET, "test-2.zarr")
S3_BUCKET_DICT = {'bibo.zarr': TEST_CUBE_1,
                  'bert.zarr': TEST_CUBE_2}


class S3BucketTest(unittest.TestCase, metaclass=ABCMeta):

    def setUp(self):
        self.maxDiff = None

    @classmethod
    def setUpClass(cls) -> None:
        rimraf(S3_BUCKET)
        os.mkdir(S3_BUCKET)
        cube = new_cube(time_periods=3,
                        variables=dict(precipitation=0.9,
                                       temperature=278.3)).chunk(dict(time=1, lat=90, lon=90))
        write_cube(cube, TEST_CUBE_1, "zarr", cube_asserted=True)
        write_cube(cube, TEST_CUBE_2, "zarr", cube_asserted=True)

    @classmethod
    def tearDownClass(cls) -> None:
        rimraf(S3_BUCKET)


class ListS3BucketKeys(S3BucketTest):
    def test_list_bucket_keys(self):
        result = [key_path for key_path in list_s3_bucket_keys(S3_BUCKET_DICT)]
        self.assertEqual(EXPECTED_KEYS, [key for key, _ in result])
        self.assertEqual(172, len(EXPECTED_KEYS))


class ListS3BucketTest(S3BucketTest, metaclass=ABCMeta):

    @abstractmethod
    def list_bucket(self, bucket_dict, **kwargs):
        pass

    @abstractmethod
    def assert_list_bucket_result(self, list_bucket_result, **kwargs):
        pass


# noinspection PyUnresolvedReferences
class ListS3BucketV12TestsMixin:

    def test_list_bucket_v12(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT)
        self.assert_list_bucket_result(list_bucket_result)
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertEqual(172, len(list_bucket_result.get('Contents')))
        self.assertNotIn('CommonPrefixes', list_bucket_result)

    def test_list_bucket_v12_prefix(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, prefix='bibo.zarr/')
        self.assert_list_bucket_result(list_bucket_result, prefix='bibo.zarr/')
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertEqual(86, len(list_bucket_result.get('Contents')))
        self.assertNotIn('CommonPrefixes', list_bucket_result)

    def test_list_bucket_v12_delimiter(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/')
        self.assertNotIn('Contents', list_bucket_result)
        self.assertIsInstance(list_bucket_result.get('CommonPrefixes'), list)
        self.assertEqual([{'Prefix': 'bert.zarr/'},
                          {'Prefix': 'bibo.zarr/'}],
                         list_bucket_result.get('CommonPrefixes'))

    def test_list_bucket_v12_delimiter_prefix(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/',
                                              prefix='bert.zarr/')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/',
                                       prefix='bert.zarr/')
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertIsInstance(list_bucket_result.get('CommonPrefixes'), list)
        self.assertEqual([{'ETag': '"5fb93bce985c60d724d9869f32e02d24"',
                           'Key': 'bert.zarr/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"16a49349c15b9cd60c5d8a05fc6ad649"',
                           'Key': 'bert.zarr/.zattrs',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 426,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"0a44cbe4de5e2936112efc2ed25b6223"',
                           'Key': 'bert.zarr/.zgroup',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 24,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"cc1a0115345e38f58a88b83722a9d2d8"',
                           'Key': 'bert.zarr/.zmetadata',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 6572,
                           'StorageClass': 'STANDARD'}],
                         list_bucket_result.get('Contents'))
        self.assertEqual([{'Prefix': 'bert.zarr/lat/'},
                          {'Prefix': 'bert.zarr/lat_bnds/'},
                          {'Prefix': 'bert.zarr/lon/'},
                          {'Prefix': 'bert.zarr/lon_bnds/'},
                          {'Prefix': 'bert.zarr/precipitation/'},
                          {'Prefix': 'bert.zarr/temperature/'},
                          {'Prefix': 'bert.zarr/time/'},
                          {'Prefix': 'bert.zarr/time_bnds/'}],
                         list_bucket_result.get('CommonPrefixes'))

    def test_list_bucket_v12_delimiter_prefix_2(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/',
                                              prefix='bert.zarr')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/',
                                       prefix='bert.zarr')
        self.assertNotIn('Contents', list_bucket_result)
        self.assertIsInstance(list_bucket_result.get('CommonPrefixes'), list)
        self.assertEqual([{'Prefix': 'bert.zarr/'}],
                         list_bucket_result.get('CommonPrefixes'))


class ListBucketV1Test(ListS3BucketTest, ListS3BucketV12TestsMixin):
    def test_list_bucket_v1_truncated(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT,
                                              max_keys=5)
        self.assert_list_bucket_result(list_bucket_result, max_keys=5, is_truncated=True,
                                       next_marker='bert.zarr/lat/.zarray')
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertEqual([{'ETag': '"5fb93bce985c60d724d9869f32e02d24"',
                           'Key': 'bert.zarr/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"16a49349c15b9cd60c5d8a05fc6ad649"',
                           'Key': 'bert.zarr/.zattrs',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 426,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"0a44cbe4de5e2936112efc2ed25b6223"',
                           'Key': 'bert.zarr/.zgroup',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 24,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"cc1a0115345e38f58a88b83722a9d2d8"',
                           'Key': 'bert.zarr/.zmetadata',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 6572,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"f64109793f6ea9ec1c9cfcb8ee97e145"',
                           'Key': 'bert.zarr/lat/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'}],
                         list_bucket_result.get('Contents'))
        self.assertNotIn('CommonPrefixes', list_bucket_result)

    def test_list_bucket_v1_result_to_xml(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/',
                                              max_keys=10, prefix='bibo.zarr/')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/',
                                       max_keys=10, prefix='bibo.zarr/')
        xml = list_bucket_result_to_xml(list_bucket_result)
        with open(os.path.join(os.path.dirname(__file__), 's3', 'list-bucket-v1-result.xml')) as fp:
            expected_xml = fp.read()
        self.assertEqual(expected_xml, xml)

    def list_bucket(self, bucket_entries: Dict[str, str], **kwargs):
        return list_s3_bucket_v1(bucket_entries,
                                 key_to_etag=True,
                                 last_modified='2019-06-24T20:43:40.862Z',
                                 **kwargs)

    def assert_list_bucket_result(self,
                                  list_bucket_result,
                                  name="s3bucket",
                                  prefix=None,
                                  delimiter=None,
                                  max_keys=1000,
                                  is_truncated=False,
                                  marker=None,
                                  next_marker=None):
        self.assertIsInstance(list_bucket_result, dict)
        self.assertEqual(name, list_bucket_result.get('Name'))
        self.assertEqual(prefix, list_bucket_result.get('Prefix'))
        self.assertEqual(delimiter, list_bucket_result.get('Delimiter'))
        self.assertEqual(max_keys, list_bucket_result.get('MaxKeys'))
        self.assertEqual(is_truncated, list_bucket_result.get('IsTruncated'))
        self.assertEqual(marker, list_bucket_result.get('Marker'))
        self.assertEqual(next_marker, list_bucket_result.get('NextMarker'))


class ListS3BucketV2Test(ListS3BucketTest, ListS3BucketV12TestsMixin):

    def test_list_bucket_v2_truncated(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT,
                                              max_keys=5)
        self.assert_list_bucket_result(list_bucket_result, max_keys=5, is_truncated=True,
                                       next_continuation_token=6)
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertEqual([{'ETag': '"5fb93bce985c60d724d9869f32e02d24"',
                           'Key': 'bert.zarr/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"16a49349c15b9cd60c5d8a05fc6ad649"',
                           'Key': 'bert.zarr/.zattrs',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 426,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"0a44cbe4de5e2936112efc2ed25b6223"',
                           'Key': 'bert.zarr/.zgroup',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 24,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"cc1a0115345e38f58a88b83722a9d2d8"',
                           'Key': 'bert.zarr/.zmetadata',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 6572,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"f64109793f6ea9ec1c9cfcb8ee97e145"',
                           'Key': 'bert.zarr/lat/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'}],
                         list_bucket_result.get('Contents'))
        self.assertNotIn('CommonPrefixes', list_bucket_result)

    def test_list_bucket_v2_result_to_xml(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/',
                                              max_keys=10, prefix='bibo.zarr/')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/',
                                       max_keys=10, prefix='bibo.zarr/')
        xml = list_bucket_result_to_xml(list_bucket_result)
        with open(os.path.join(os.path.dirname(__file__), 's3', 'list-bucket-v2-result.xml')) as fp:
            expected_xml = fp.read()
        self.assertEqual(expected_xml, xml)

    def list_bucket(self, bucket_entries: Dict[str, str], **kwargs):
        return list_s3_bucket_v2(bucket_entries,
                                 key_to_etag=True,
                                 last_modified='2019-06-24T20:43:40.862Z',
                                 **kwargs)

    def assert_list_bucket_result(self,
                                  list_bucket_result,
                                  name="s3bucket",
                                  prefix=None,
                                  delimiter=None, max_keys=1000,
                                  is_truncated=False,
                                  start_after=None,
                                  continuation_token=None,
                                  next_continuation_token=None):
        self.assertIsInstance(list_bucket_result, dict)
        self.assertEqual(name, list_bucket_result.get('Name'))
        self.assertEqual(prefix, list_bucket_result.get('Prefix'))
        self.assertEqual(delimiter, list_bucket_result.get('Delimiter'))
        self.assertEqual(max_keys, list_bucket_result.get('MaxKeys'))
        self.assertEqual(is_truncated, list_bucket_result.get('IsTruncated'))
        self.assertEqual(start_after, list_bucket_result.get('Marker'))
        self.assertEqual(continuation_token, list_bucket_result.get('ContinuationToken'))
        self.assertEqual(next_continuation_token, list_bucket_result.get('NextContinuationToken'))


EXPECTED_KEYS = [
    'bert.zarr/',
    'bert.zarr/.zattrs',
    'bert.zarr/.zgroup',
    'bert.zarr/.zmetadata',
    'bert.zarr/lat/',
    'bert.zarr/lat/.zarray',
    'bert.zarr/lat/.zattrs',
    'bert.zarr/lat/0',
    'bert.zarr/lat_bnds/',
    'bert.zarr/lat_bnds/.zarray',
    'bert.zarr/lat_bnds/.zattrs',
    'bert.zarr/lat_bnds/0.0',
    'bert.zarr/lat_bnds/1.0',
    'bert.zarr/lon/',
    'bert.zarr/lon/.zarray',
    'bert.zarr/lon/.zattrs',
    'bert.zarr/lon/0',
    'bert.zarr/lon_bnds/',
    'bert.zarr/lon_bnds/.zarray',
    'bert.zarr/lon_bnds/.zattrs',
    'bert.zarr/lon_bnds/0.0',
    'bert.zarr/lon_bnds/1.0',
    'bert.zarr/lon_bnds/2.0',
    'bert.zarr/lon_bnds/3.0',
    'bert.zarr/precipitation/',
    'bert.zarr/precipitation/.zarray',
    'bert.zarr/precipitation/.zattrs',
    'bert.zarr/precipitation/0.0.0',
    'bert.zarr/precipitation/0.0.1',
    'bert.zarr/precipitation/0.0.2',
    'bert.zarr/precipitation/0.0.3',
    'bert.zarr/precipitation/0.1.0',
    'bert.zarr/precipitation/0.1.1',
    'bert.zarr/precipitation/0.1.2',
    'bert.zarr/precipitation/0.1.3',
    'bert.zarr/precipitation/1.0.0',
    'bert.zarr/precipitation/1.0.1',
    'bert.zarr/precipitation/1.0.2',
    'bert.zarr/precipitation/1.0.3',
    'bert.zarr/precipitation/1.1.0',
    'bert.zarr/precipitation/1.1.1',
    'bert.zarr/precipitation/1.1.2',
    'bert.zarr/precipitation/1.1.3',
    'bert.zarr/precipitation/2.0.0',
    'bert.zarr/precipitation/2.0.1',
    'bert.zarr/precipitation/2.0.2',
    'bert.zarr/precipitation/2.0.3',
    'bert.zarr/precipitation/2.1.0',
    'bert.zarr/precipitation/2.1.1',
    'bert.zarr/precipitation/2.1.2',
    'bert.zarr/precipitation/2.1.3',
    'bert.zarr/temperature/',
    'bert.zarr/temperature/.zarray',
    'bert.zarr/temperature/.zattrs',
    'bert.zarr/temperature/0.0.0',
    'bert.zarr/temperature/0.0.1',
    'bert.zarr/temperature/0.0.2',
    'bert.zarr/temperature/0.0.3',
    'bert.zarr/temperature/0.1.0',
    'bert.zarr/temperature/0.1.1',
    'bert.zarr/temperature/0.1.2',
    'bert.zarr/temperature/0.1.3',
    'bert.zarr/temperature/1.0.0',
    'bert.zarr/temperature/1.0.1',
    'bert.zarr/temperature/1.0.2',
    'bert.zarr/temperature/1.0.3',
    'bert.zarr/temperature/1.1.0',
    'bert.zarr/temperature/1.1.1',
    'bert.zarr/temperature/1.1.2',
    'bert.zarr/temperature/1.1.3',
    'bert.zarr/temperature/2.0.0',
    'bert.zarr/temperature/2.0.1',
    'bert.zarr/temperature/2.0.2',
    'bert.zarr/temperature/2.0.3',
    'bert.zarr/temperature/2.1.0',
    'bert.zarr/temperature/2.1.1',
    'bert.zarr/temperature/2.1.2',
    'bert.zarr/temperature/2.1.3',
    'bert.zarr/time/',
    'bert.zarr/time/.zarray',
    'bert.zarr/time/.zattrs',
    'bert.zarr/time/0',
    'bert.zarr/time_bnds/',
    'bert.zarr/time_bnds/.zarray',
    'bert.zarr/time_bnds/.zattrs',
    'bert.zarr/time_bnds/0.0',
    'bibo.zarr/',
    'bibo.zarr/.zattrs',
    'bibo.zarr/.zgroup',
    'bibo.zarr/.zmetadata',
    'bibo.zarr/lat/',
    'bibo.zarr/lat/.zarray',
    'bibo.zarr/lat/.zattrs',
    'bibo.zarr/lat/0',
    'bibo.zarr/lat_bnds/',
    'bibo.zarr/lat_bnds/.zarray',
    'bibo.zarr/lat_bnds/.zattrs',
    'bibo.zarr/lat_bnds/0.0',
    'bibo.zarr/lat_bnds/1.0',
    'bibo.zarr/lon/',
    'bibo.zarr/lon/.zarray',
    'bibo.zarr/lon/.zattrs',
    'bibo.zarr/lon/0',
    'bibo.zarr/lon_bnds/',
    'bibo.zarr/lon_bnds/.zarray',
    'bibo.zarr/lon_bnds/.zattrs',
    'bibo.zarr/lon_bnds/0.0',
    'bibo.zarr/lon_bnds/1.0',
    'bibo.zarr/lon_bnds/2.0',
    'bibo.zarr/lon_bnds/3.0',
    'bibo.zarr/precipitation/',
    'bibo.zarr/precipitation/.zarray',
    'bibo.zarr/precipitation/.zattrs',
    'bibo.zarr/precipitation/0.0.0',
    'bibo.zarr/precipitation/0.0.1',
    'bibo.zarr/precipitation/0.0.2',
    'bibo.zarr/precipitation/0.0.3',
    'bibo.zarr/precipitation/0.1.0',
    'bibo.zarr/precipitation/0.1.1',
    'bibo.zarr/precipitation/0.1.2',
    'bibo.zarr/precipitation/0.1.3',
    'bibo.zarr/precipitation/1.0.0',
    'bibo.zarr/precipitation/1.0.1',
    'bibo.zarr/precipitation/1.0.2',
    'bibo.zarr/precipitation/1.0.3',
    'bibo.zarr/precipitation/1.1.0',
    'bibo.zarr/precipitation/1.1.1',
    'bibo.zarr/precipitation/1.1.2',
    'bibo.zarr/precipitation/1.1.3',
    'bibo.zarr/precipitation/2.0.0',
    'bibo.zarr/precipitation/2.0.1',
    'bibo.zarr/precipitation/2.0.2',
    'bibo.zarr/precipitation/2.0.3',
    'bibo.zarr/precipitation/2.1.0',
    'bibo.zarr/precipitation/2.1.1',
    'bibo.zarr/precipitation/2.1.2',
    'bibo.zarr/precipitation/2.1.3',
    'bibo.zarr/temperature/',
    'bibo.zarr/temperature/.zarray',
    'bibo.zarr/temperature/.zattrs',
    'bibo.zarr/temperature/0.0.0',
    'bibo.zarr/temperature/0.0.1',
    'bibo.zarr/temperature/0.0.2',
    'bibo.zarr/temperature/0.0.3',
    'bibo.zarr/temperature/0.1.0',
    'bibo.zarr/temperature/0.1.1',
    'bibo.zarr/temperature/0.1.2',
    'bibo.zarr/temperature/0.1.3',
    'bibo.zarr/temperature/1.0.0',
    'bibo.zarr/temperature/1.0.1',
    'bibo.zarr/temperature/1.0.2',
    'bibo.zarr/temperature/1.0.3',
    'bibo.zarr/temperature/1.1.0',
    'bibo.zarr/temperature/1.1.1',
    'bibo.zarr/temperature/1.1.2',
    'bibo.zarr/temperature/1.1.3',
    'bibo.zarr/temperature/2.0.0',
    'bibo.zarr/temperature/2.0.1',
    'bibo.zarr/temperature/2.0.2',
    'bibo.zarr/temperature/2.0.3',
    'bibo.zarr/temperature/2.1.0',
    'bibo.zarr/temperature/2.1.1',
    'bibo.zarr/temperature/2.1.2',
    'bibo.zarr/temperature/2.1.3',
    'bibo.zarr/time/',
    'bibo.zarr/time/.zarray',
    'bibo.zarr/time/.zattrs',
    'bibo.zarr/time/0',
    'bibo.zarr/time_bnds/',
    'bibo.zarr/time_bnds/.zarray',
    'bibo.zarr/time_bnds/.zattrs',
    'bibo.zarr/time_bnds/0.0'
]


# test/webapi/test_service.py

import re
import unittest

from xcube.webapi.service import url_pattern, new_default_config


class DefaultConfigTest(unittest.TestCase):
    def test_new_default_config(self):
        config = new_default_config(["/home/bibo/data/cube-1.zarr",
                                     "/home/bibo/data/cube-2.nc"],
                                    dict(conc_chl=(0.0, 20.0),
                                         conc_tsm=(0.0, 12.0, 'plasma')))
        self.assertEqual({
            'Datasets': [
                {
                    'FileSystem': 'file',
                    'Format': 'zarr',
                    'Identifier': 'dataset_1',
                    'Path': '/home/bibo/data/cube-1.zarr',
                    'Title': 'cube-1.zarr'
                },
                {
                    'FileSystem': 'file',
                    'Format': 'netcdf4',
                    'Identifier': 'dataset_2',
                    'Path': '/home/bibo/data/cube-2.nc',
                    'Title': 'cube-2.nc'
                }
            ],
            'Styles': [
                {'Identifier': 'default',
                 'ColorMappings': {
                     'conc_chl': {'ValueRange': [0.0, 20.0]},
                     'conc_tsm': {'ColorBar': 'plasma',
                                  'ValueRange': [0.0, 12.0]}},
                 }
            ]},
            config)

        with self.assertRaises(ValueError) as cm:
            new_default_config(["/home/bibo/data/cube-1.zarr",
                                "/home/bibo/data/cube-2.nc"],
                               dict(conc_chl=20.0,
                                    conc_tsm=(0.0, 12.0, 'plasma')))
        self.assertEqual("illegal style: conc_chl=20.0", f"{cm.exception}")


class UrlPatternTest(unittest.TestCase):
    def test_url_pattern_works(self):
        re_pattern = url_pattern('/open/{{id1}}ws/{{id2}}wf')
        matcher = re.fullmatch(re_pattern, '/open/34ws/a66wf')
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'id1': '34', 'id2': 'a66'})

        re_pattern = url_pattern('/open/ws{{id1}}/wf{{id2}}')
        matcher = re.fullmatch(re_pattern, '/open/ws34/wfa66')
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'id1': '34', 'id2': 'a66'})

        re_pattern = url_pattern('/datasets/{{ds_id}}/data.zarr/(?P<path>.*)')

        matcher = re.fullmatch(re_pattern, '/datasets/S2PLUS_2017/data.zarr/')
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'ds_id': 'S2PLUS_2017', 'path': ''})

        matcher = re.fullmatch(re_pattern, '/datasets/S2PLUS_2017/data.zarr/conc_chl/.zattrs')
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'ds_id': 'S2PLUS_2017', 'path': 'conc_chl/.zattrs'})

        x = 'C%3A%5CUsers%5CNorman%5CIdeaProjects%5Cccitools%5Cect-core%5Ctest%5Cui%5CTEST_WS_3'
        re_pattern = url_pattern('/ws/{{base_dir}}/res/{{res_name}}/add')
        matcher = re.fullmatch(re_pattern, '/ws/%s/res/SST/add' % x)
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'base_dir': x, 'res_name': 'SST'})

    def test_url_pattern_ok(self):
        self.assertEqual('/version',
                         url_pattern('/version'))
        self.assertEqual(r'(?P<num>[^\;\/\?\:\@\&\=\+\$\,]+)/get',
                         url_pattern('{{num}}/get'))
        self.assertEqual(r'/open/(?P<ws_name>[^\;\/\?\:\@\&\=\+\$\,]+)',
                         url_pattern('/open/{{ws_name}}'))
        self.assertEqual(r'/open/ws(?P<id1>[^\;\/\?\:\@\&\=\+\$\,]+)/wf(?P<id2>[^\;\/\?\:\@\&\=\+\$\,]+)',
                         url_pattern('/open/ws{{id1}}/wf{{id2}}'))
        self.assertEqual(r'/datasets/(?P<ds_id>[^\;\/\?\:\@\&\=\+\$\,]+)/data.zip/(.*)',
                         url_pattern('/datasets/{{ds_id}}/data.zip/(.*)'))

    def test_url_pattern_fail(self):
        with self.assertRaises(ValueError) as cm:
            url_pattern('/open/{{ws/name}}')
        self.assertEqual(str(cm.exception), 'name in {{name}} must be a valid identifier, but got "ws/name"')

        with self.assertRaises(ValueError) as cm:
            url_pattern('/info/{{id}')
        self.assertEqual(str(cm.exception), 'no matching "}}" after "{{" in "/info/{{id}"')


# test/webapi/controllers/test_catalogue.py

import os.path
import unittest
from typing import Any, Optional

from test.webapi.helpers import new_test_service_context
from xcube.webapi.context import ServiceContext
from xcube.webapi.controllers.catalogue import get_color_bars
from xcube.webapi.controllers.catalogue import get_dataset
from xcube.webapi.controllers.catalogue import get_datasets
from xcube.webapi.errors import ServiceBadRequestError, ServiceAuthError


def new_ctx():
    data_store_root = os.path.join(
        os.path.dirname(__file__),
        '..', '..', '..', 'examples', 'serve', 'demo'
    )
    config = dict(
        Authentication=dict(
            Domain='xcube-dev.eu.auth0.com',
            Audience='https://xcube-dev/api/',
        ),
        AccessControl=dict(
            RequiredScopes=[
                'read:dataset:{Dataset}',
                'read:variable:{Variable}'
            ]
        ),
        DataStores=[
            dict(
                Identifier='local',
                StoreId='file',
                StoreParams=dict(
                    root=data_store_root
                ),
                Datasets=[
                    # "base" will only appear for unauthorized clients
                    dict(
                        Identifier='base',
                        Path='cube-1-250-250.zarr',
                        TimeSeriesDataset='local~cube-5-100-200.zarr',
                        AccessControl=dict(
                            IsSubstitute=True
                        )
                    ),
                    # "time_series" will not appear at all,
                    # because it is a "hidden" resource
                    dict(
                        Identifier='time_chunked',
                        Path='cube-5-100-200.zarr',
                        Hidden=True,
                    ),
                ]
            ),
            dict(
                Identifier='remote',
                StoreId='s3',
                StoreParams=dict(
                    root='xcube-examples',
                    storage_options=dict(anon=True),
                ),
                Datasets=[
                    dict(
                        Identifier='base',
                        Path='OLCI-SNS-RAW-CUBE-2.zarr'
                    ),
                ]
            )
        ],
        Datasets=[
            # "local" will only appear for unauthorized clients
            dict(
                Identifier='local_base_1w',
                FileSystem='memory',
                Path='examples/serve/demo/resample_in_time.py',
                Function='compute_dataset',
                InputDatasets=['local~cube-1-250-250.zarr'],
                InputParameters=dict(
                    period='1W',
                    incl_stdev=True
                ),
                AccessControl=dict(
                    IsSubstitute=True
                )
            ),
            dict(
                Identifier='remote_base_1w',
                FileSystem='memory',
                Path='examples/serve/demo/resample_in_time.py',
                Function='compute_dataset',
                InputDatasets=['remote~OLCI-SNS-RAW-CUBE-2.zarr'],
                InputParameters=dict(
                    period='1W',
                    incl_stdev=True
                ),
            ),
        ],
    )
    return ServiceContext(config=config)


class CatalogueControllerTest(unittest.TestCase):
    def test_unauthorized_access(self):
        ctx = new_ctx()
        granted_scopes = None
        response = get_datasets(ctx, granted_scopes=granted_scopes)
        datasets = self._assert_datasets(response)
        datasets_dict = {ds['id']: ds for ds in datasets}
        self.assertEqual(
            {
                'local~cube-1-250-250.zarr',
                'local_base_1w',
                # Not selected, because they require authorisation
                # 'remote~OLCI-SNS-RAW-CUBE-2.zarr',
                # 'remote_base_1w',
            },
            set(datasets_dict)
        )

        dataset = get_dataset(ctx, 'local~cube-1-250-250.zarr')
        self.assertIn('variables', dataset)
        var_dict = {v['name']: v for v in dataset['variables']}
        self.assertEqual({'c2rcc_flags',
                          'conc_tsm',
                          'kd489',
                          'conc_chl',
                          'quality_flags'},
                         set(var_dict.keys()))

        dataset = get_dataset(ctx, 'local_base_1w')
        self.assertIn('variables', dataset)
        var_dict = {v['name']: v for v in dataset['variables']}
        self.assertEqual({'c2rcc_flags',
                          'c2rcc_flags_stdev',
                          'conc_chl',
                          'conc_chl_stdev',
                          'conc_tsm',
                          'conc_tsm_stdev',
                          'kd489',
                          'kd489_stdev',
                          'quality_flags',
                          'quality_flags_stdev'},
                         set(var_dict.keys()))

        with self.assertRaises(ServiceAuthError) as cm:
            get_dataset(ctx, 'remote_base_1w')
        self.assertEqual(
            'HTTP 401: Missing permission'
            ' (Missing permission'
            ' read:dataset:remote_base_1w)',
            f'{cm.exception}'
        )

        with self.assertRaises(ServiceAuthError) as cm:
            get_dataset(ctx, 'remote~OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual(
            'HTTP 401: Missing permission'
            ' (Missing permission'
            ' read:dataset:remote~OLCI-SNS-RAW-CUBE-2.zarr)',
            f'{cm.exception}'
        )

    def test_authorized_access_with_joker_scopes(self):
        ctx = new_ctx()
        granted_scopes = {
            'read:dataset:*',
            'read:variable:*'
        }
        response = get_datasets(ctx, granted_scopes=granted_scopes)
        datasets = self._assert_datasets(response)
        datasets_dict = {ds['id']: ds for ds in datasets}
        self.assertEqual(
            {
                'local_base_1w',
                'local~cube-1-250-250.zarr',
                'remote_base_1w',
                'remote~OLCI-SNS-RAW-CUBE-2.zarr',
            },
            set(datasets_dict)
        )

    def test_authorized_access_with_specific_scopes(self):
        ctx = new_ctx()
        granted_scopes = {
            'read:dataset:remote*',
            'read:variable:*'
        }
        response = get_datasets(ctx, granted_scopes=granted_scopes)
        datasets = self._assert_datasets(response)
        datasets_dict = {ds['id']: ds for ds in datasets}
        self.assertEqual(
            {
                # Not selected, because they are substitutes
                # 'local_base_1w',
                # 'local~cube-1-250-250.zarr',
                'remote_base_1w',
                'remote~OLCI-SNS-RAW-CUBE-2.zarr',
            },
            set(datasets_dict)
        )

    def test_datasets(self):
        ctx = new_test_service_context()
        response = get_datasets(ctx)
        datasets = self._assert_datasets(response, expected_count=2)
        for dataset in datasets:
            self.assertIsInstance(dataset, dict)
            self.assertIn("id", dataset)
            self.assertIn("title", dataset)
            self.assertNotIn("variables", dataset)
            self.assertNotIn("dimensions", dataset)
            self.assertNotIn("rgbSchema", dataset)

    def test_dataset_with_details(self):
        ctx = new_test_service_context()
        response = get_datasets(ctx, details=True, base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=2)

        demo_dataset = None
        demo_1w_dataset = None
        for dataset in datasets:
            self.assertIsInstance(dataset, dict)
            self.assertIn("id", dataset)
            self.assertIn("title", dataset)
            self.assertIn("attributions", dataset)
            self.assertIn("variables", dataset)
            self.assertIn("dimensions", dataset)
            self.assertNotIn("rgbSchema", dataset)
            if dataset["id"] == "demo":
                demo_dataset = dataset
            if dataset["id"] == "demo-1w":
                demo_1w_dataset = dataset

        self.assertIsNotNone(demo_dataset)
        self.assertIsNotNone(demo_1w_dataset)
        self.assertEqual(["© by EU H2020 CyanoAlert project"], demo_dataset['attributions'])
        self.assertEqual(["© by Brockmann Consult GmbH 2020, "
                          "contains modified Copernicus Data 2019, processed by ESA"], demo_1w_dataset['attributions'])

    def test_dataset_with_details_and_rgb_schema(self):
        ctx = new_test_service_context('config-rgb.yml')
        response = get_datasets(ctx, details=True, base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=1)
        dataset = datasets[0]
        self.assertIsInstance(dataset, dict)
        self.assertEqual(
            {
                'varNames': ['conc_chl', 'conc_tsm', 'kd489'],
                'normRanges': [(0.0, 24.0), (0.0, 100.0),
                               (0.0, 6.0)],
                'tileUrl': 'http://test/datasets/demo-rgb/vars/rgb/tiles2/{z}/{y}/{x}',
            },
            dataset.get("rgbSchema")
        )
        response = get_datasets(ctx, details=True, client='ol4',
                                base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=1)
        dataset = datasets[0]
        self.assertIsInstance(dataset, dict)
        self.assertEqual(
            {
                'varNames': ['conc_chl', 'conc_tsm', 'kd489'],
                'normRanges': [(0.0, 24.0), (0.0, 100.0),
                               (0.0, 6.0)],
                'tileUrl': 'http://test/datasets/demo-rgb/vars/rgb/tiles2/{z}/{y}/{x}',
                'tileSourceOptions': {
                    'url': 'http://test/datasets/demo-rgb/vars/rgb/tiles/{z}/{x}/{y}.png',
                    'projection': 'EPSG:4326',
                    'tileGrid': {'extent': [0, 50, 5, 52.5],
                                 'origin': [0, 52.5],
                                 'resolutions': [0.01, 0.005,
                                                 0.0025],
                                 'sizes': [[2, 1], [4, 2], [8, 4]],
                                 'tileSize': [250, 250]}
                }
            },
            dataset.get("rgbSchema")
        )

    def test_dataset_with_point(self):
        ctx = new_test_service_context()

        response = get_datasets(ctx, point=(1.7, 51.2),
                                base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=2)
        dataset = datasets[0]
        self.assertIsInstance(dataset, dict)
        self.assertIn("id", dataset)
        self.assertIn("title", dataset)
        self.assertNotIn("variables", dataset)
        self.assertNotIn("dimensions", dataset)

        response = get_datasets(ctx, point=(1.7, 58.0),
                                base_url="http://test")
        self._assert_datasets(response, expected_count=0)

    def test_dataset_with_point_and_details(self):
        ctx = new_test_service_context()

        response = get_datasets(ctx, point=(1.7, 51.2), details=True,
                                base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=2)
        dataset = datasets[0]
        self.assertIsInstance(dataset, dict)
        self.assertIn("id", dataset)
        self.assertIn("title", dataset)
        self.assertIn("variables", dataset)
        self.assertIn("dimensions", dataset)

        response = get_datasets(ctx, point=(1.7, 58.0), details=True,
                                base_url="http://test")
        self._assert_datasets(response, 0)

    def test_get_colorbars(self):
        ctx = ServiceContext()

        response = get_color_bars(ctx, 'application/json')
        self.assertIsInstance(response, str)
        self.assertTrue(len(response) > 40)
        self.assertEqual('[\n  [\n    "Perceptually Uniform Sequenti',
                         response[0:40])

        response = get_color_bars(ctx, 'text/html')
        self.assertIsInstance(response, str)
        self.assertTrue(len(response) > 40)
        self.assertEqual('<!DOCTYPE html>\n<html lang="en">\n<head><',
                         response[0:40])

        with self.assertRaises(ServiceBadRequestError) as cm:
            get_color_bars(ctx, 'text/xml')
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual("Format 'text/xml' not supported for color bars",
                         cm.exception.reason)

    def _assert_datasets(self,
                         response: Any,
                         expected_count: Optional[int] = None):
        self.assertIsInstance(response, dict)
        datasets = response.get("datasets")
        self.assertIsInstance(datasets, list)
        if expected_count is not None:
            self.assertEqual(expected_count, len(datasets))
        return datasets


# test/webapi/controllers/test_places.py

import unittest

from test.webapi.helpers import new_test_service_context
from xcube.webapi.controllers.places import find_places, find_dataset_places
from xcube.webapi.errors import ServiceBadRequestError


class FeaturesControllerTest(unittest.TestCase):

    def test_find_places_all(self):
        ctx = new_test_service_context()
        places = find_places(ctx, "all", "http://localhost:9090")
        self._assertPlaceGroup(places, 6, {'0', '1', '2', '3', '4', '5'})

    def test_find_places_by_box(self):
        ctx = new_test_service_context()
        places = find_places(ctx, "all", "http://localhost:9090", box_coords="-1,49,2,55")
        self._assertPlaceGroup(places, 2, {'0', '3'})

        with self.assertRaises(ServiceBadRequestError) as cm:
            find_places(ctx, "all", "http://localhost:9090", box_coords="-1,49,55")
        self.assertEqual("HTTP 400: Received invalid bounding box geometry", f"{cm.exception}")

        with self.assertRaises(ServiceBadRequestError) as cm:
            find_places(ctx, "all", "http://localhost:9090", box_coords="-1,49,x,55")
        self.assertEqual("HTTP 400: Received invalid bounding box geometry", f"{cm.exception}")

    def test_find_places_by_wkt(self):
        ctx = new_test_service_context()
        places = find_places(ctx, "all", "http://localhost:9090", geom_wkt="POLYGON ((2 49, 2 55, -1 55, -1 49, 2 49))")
        self._assertPlaceGroup(places, 2, {'0', '3'})

        with self.assertRaises(ServiceBadRequestError) as cm:
            find_places(ctx, "all", "http://localhost:9090", geom_wkt="POLYGLON ((2 49, 2 55, -1 55, -1 49, 2 49))")
        self.assertEqual("HTTP 400: Received invalid geometry WKT", f"{cm.exception}")

    def test_find_places_by_geojson(self):
        ctx = new_test_service_context()

        geojson_obj = {'type': 'Polygon',
                       'coordinates': (((2.0, 49.0), (2.0, 55.0), (-1.0, 55.0), (-1.0, 49.0), (2.0, 49.0)),)}
        places = find_places(ctx, "all", "http://localhost:9090", geojson_obj=geojson_obj)
        self._assertPlaceGroup(places, 2, {'0', '3'})

        geojson_obj = {'type': 'Feature', 'properties': {}, 'geometry': geojson_obj}
        places = find_places(ctx, "all", "http://localhost:9090", geojson_obj=geojson_obj)
        self._assertPlaceGroup(places, 2, {'0', '3'})

        geojson_obj = {'type': 'FeatureCollection', 'features': [geojson_obj]}
        places = find_places(ctx, "all", "http://localhost:9090", geojson_obj=geojson_obj)
        self._assertPlaceGroup(places, 2, {'0', '3'})

        with self.assertRaises(ServiceBadRequestError) as cm:
            geojson_obj = {'type': 'FeatureCollection', 'features': []}
            find_places(ctx, "all", "http://localhost:9090", geojson_obj=geojson_obj)
        self.assertEqual("HTTP 400: Received invalid GeoJSON object", f"{cm.exception}")

    def test_find_dataset_features(self):
        ctx = new_test_service_context()
        places = find_dataset_places(ctx, "all", "demo", "http://localhost:8080")
        self._assertPlaceGroup(places, 3, {'0', '1', '2'})

    def _assertPlaceGroup(self, feature_collection, expected_count, expected_ids):
        self.assertIsInstance(feature_collection, dict)
        self.assertIn("type", feature_collection)
        self.assertEqual("FeatureCollection", feature_collection["type"])
        self.assertIn("features", feature_collection)
        features = feature_collection["features"]
        self.assertIsInstance(features, list)
        self.assertEqual(expected_count, len(features))
        actual_ids = {f["id"] for f in features if "id" in f}
        self.assertEqual(expected_ids, actual_ids)


# test/webapi/controllers/test_tiles.py

import unittest

from test.webapi.helpers import new_test_service_context, RequestParamsMock
from xcube.webapi.controllers.tiles import get_dataset_tile, get_dataset_tile_grid, \
    get_legend
from xcube.webapi.errors import ServiceBadRequestError, ServiceResourceNotFoundError


class TilesControllerTest(unittest.TestCase):

    def test_get_dataset_tile(self):
        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock())
        self.assertIsInstance(tile, bytes)

        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '-20', '0', '0', RequestParamsMock())
        self.assertIsInstance(tile, bytes)

    def test_get_dataset_tile_invalid_dataset(self):
        ctx = new_test_service_context()
        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            get_dataset_tile(ctx, 'demo-rgb', 'conc_tsm', '0', '0', '0', RequestParamsMock())
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Dataset "demo-rgb" not found', f'{cm.exception.reason}')

    def test_get_dataset_tile_invalid_variable(self):
        ctx = new_test_service_context()
        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            get_dataset_tile(ctx, 'demo', 'conc_tdi', '0', '0', '0', RequestParamsMock())
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Variable "conc_tdi" not found in dataset "demo"', f'{cm.exception.reason}')

    def test_get_dataset_tile_with_all_params(self):
        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='current', cbar='plasma',
                                                                                          vmin='0.1', vmax='0.3'))
        self.assertIsInstance(tile, bytes)

    def test_get_dataset_tile_with_time_dim(self):
        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='2017-01-26'))
        self.assertIsInstance(tile, bytes)

        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='2017-01-26/2017-01-27'))
        self.assertIsInstance(tile, bytes)

        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='current'))
        self.assertIsInstance(tile, bytes)

    def test_get_dataset_tile_with_invalid_time_dim(self):
        ctx = new_test_service_context()
        with self.assertRaises(ServiceBadRequestError) as cm:
            get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='Gnaaark!'))
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual("'Gnaaark!' is not a valid value for dimension 'time'",
                         cm.exception.reason)

    def test_get_dataset_rgb_tile(self):
        ctx = new_test_service_context('config-rgb.yml')
        tile = get_dataset_tile(ctx, 'demo-rgb', 'rgb', '0', '0', '0', RequestParamsMock())
        self.assertIsInstance(tile, bytes)

    def test_get_dataset_rgb_tile_invalid_b(self):
        ctx = new_test_service_context('config-rgb.yml')
        with self.assertRaises(ServiceBadRequestError) as cm:
            get_dataset_tile(ctx, 'demo-rgb', 'rgb', '0', '0', '0', RequestParamsMock(b='refl_3'))
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual("Variable 'refl_3' not found in dataset 'demo-rgb'",
                         cm.exception.reason)

    def test_get_dataset_rgb_tile_no_vars(self):
        ctx = new_test_service_context()
        with self.assertRaises(ServiceBadRequestError) as cm:
            get_dataset_tile(ctx, 'demo', 'rgb', '0', '0', '0', RequestParamsMock())
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual("No variable in dataset 'demo' specified for RGB",
                         cm.exception.reason)

    def test_get_dataset_tile_grid(self):
        self.maxDiff = None

        ctx = new_test_service_context()
        tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol4', 'http://bibo')
        self.assertEqual(
            {
                'projection': 'EPSG:4326',
                'tileGrid': {'extent': [0, 50, 5, 52.5],
                             'origin': [0, 52.5],
                             'resolutions': [0.01, 0.005, 0.0025],
                             'sizes': [[2, 1], [4, 2], [8, 4]],
                             'tileSize': [250, 250]},
                'url': 'http://bibo/datasets/demo/vars/conc_chl/tiles2/{z}/{y}/{x}'
            },
            tile_grid)

        with self.assertRaises(NotImplementedError):
            get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'cesium', 'http://bibo')

        with self.assertRaises(ServiceBadRequestError) as cm:
            get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol2.json', 'http://bibo')
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual('Unknown tile client "ol2.json"', cm.exception.reason)

    def test_get_dataset_tile_grid_with_prefix(self):
        self.maxDiff = None

        ctx = new_test_service_context(prefix='api/v1')

        tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol4', 'http://bibo')
        self.assertEqual(
            {
                'projection': 'EPSG:4326',
                'tileGrid': {'extent': [0, 50, 5.0, 52.5],
                             'origin': [0, 52.5],
                             'resolutions': [0.01, 0.005, 0.0025],
                             'sizes': [[2, 1], [4, 2], [8, 4]],
                             'tileSize': [250, 250]},
                'url': 'http://bibo/api/v1/datasets/demo/vars/conc_chl/tiles2/{z}/{y}/{x}'
            },
            tile_grid)

    def test_get_legend(self):
        ctx = new_test_service_context()
        image = get_legend(ctx, 'demo', 'conc_chl', RequestParamsMock())
        self.assertEqual("<class 'bytes'>", str(type(image)))

        # This is fine, because we fall back to "viridis".
        image = get_legend(ctx, 'demo', 'conc_chl', RequestParamsMock(cbar='sun-shine'))
        self.assertEqual("<class 'bytes'>", str(type(image)))

        with self.assertRaises(ServiceBadRequestError) as cm:
            get_legend(ctx, 'demo', 'conc_chl', RequestParamsMock(vmin='sun-shine'))
        self.assertEqual("""Parameter "vmin" must be a number, but was 'sun-shine'""", cm.exception.reason)

        with self.assertRaises(ServiceBadRequestError) as cm:
            get_legend(ctx, 'demo', 'conc_chl', RequestParamsMock(width='sun-shine'))
        self.assertEqual("""Parameter "width" must be an integer, but was 'sun-shine'""", cm.exception.reason)

    @property
    def base_url(self):
        return f'http://bibo'


# test/webapi/controllers/test_timeseries.py

import json
import os
import unittest

import numpy as np
import pandas as pd
import xarray as xr

from test.mixins import AlmostEqualDeepMixin
from test.webapi.helpers import new_test_service_context
from xcube.webapi.context import ServiceContext
from xcube.webapi.controllers.timeseries import get_time_series, _collect_timeseries_result


class TimeSeriesControllerTest(unittest.TestCase, AlmostEqualDeepMixin):

    def setUp(self) -> None:
        self.maxDiff = None

    def test_get_time_series_for_point(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Point", coordinates=[2.1, 51.4]),
                                        start_date=np.datetime64('2017-01-15'),
                                        end_date=np.datetime64('2017-01-29'))
        expected_result = [{'mean': 3.534773588180542, 'time': '2017-01-16T10:09:22Z'},
                           {'mean': None, 'time': '2017-01-25T09:35:51Z'},
                           {'mean': None, 'time': '2017-01-26T10:50:17Z'},
                           {'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_out_of_bounds(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Point", coordinates=[-150.0, -30.0]))
        expected_result = []
        self.assertEqual(expected_result, actual_result)

    def test_get_time_series_for_point_one_valid(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Point", coordinates=[2.1, 51.4]),
                                        agg_methods=['mean', 'count'],
                                        start_date=np.datetime64('2017-01-15'),
                                        end_date=np.datetime64('2017-01-29'),
                                        max_valids=1)
        expected_result = [{'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_only_valids(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Point", coordinates=[2.1, 51.4]),
                                        start_date=np.datetime64('2017-01-15'),
                                        end_date=np.datetime64('2017-01-29'),
                                        max_valids=-1)
        expected_result = [{'mean': 3.534773588180542, 'time': '2017-01-16T10:09:22Z'},
                           {'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Polygon", coordinates=[[
                                            [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                        ]]),
                                        agg_methods=['mean', 'count'])
        expected_result = [{'count': 122392,
                            'count_tot': 159600,
                            'mean': 56.12519223634024,
                            'time': '2017-01-16T10:09:22Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'mean': None,
                            'time': '2017-01-25T09:35:51Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'mean': None,
                            'time': '2017-01-26T10:50:17Z'},
                           {'count': 132066,
                            'count_tot': 159600,
                            'mean': 49.70755256053988,
                            'time': '2017-01-28T09:58:11Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'mean': None,
                            'time': '2017-01-30T10:46:34Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon_with_all_agg_methods(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Polygon", coordinates=[[
                                            [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                        ]]),
                                        agg_methods=['mean', 'median', 'std', 'min', 'max', 'count'])

        expected_result = [{'count': 122392,
                            'count_tot': 159600,
                            'max': 166.57278442382812,
                            'mean': 56.12519223634024,
                            'median': 48.009796142578125,
                            'min': 0.02251400426030159,
                            'std': 40.78859862094861,
                            'time': '2017-01-16T10:09:22Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'max': None,
                            'mean': None,
                            'median': None,
                            'min': None,
                            'std': None,
                            'time': '2017-01-25T09:35:51Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'max': None,
                            'mean': None,
                            'median': None,
                            'min': None,
                            'std': None,
                            'time': '2017-01-26T10:50:17Z'},
                           {'count': 132066,
                            'count_tot': 159600,
                            'max': 158.7908477783203,
                            'mean': 49.70755256053988,
                            'median': 39.326446533203125,
                            'min': 0.02607600949704647,
                            'std': 34.98868194514787,
                            'time': '2017-01-28T09:58:11Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'max': None,
                            'mean': None,
                            'median': None,
                            'min': None,
                            'std': None,
                            'time': '2017-01-30T10:46:34Z'}]

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon_one_valid(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Polygon", coordinates=[[
                                            [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                        ]]),
                                        agg_methods=['mean', 'count'],
                                        max_valids=1)
        expected_result = [{'count': 132066,
                            'count_tot': 159600,
                            'mean': 49.70755256053988,
                            'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon_only_valids(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Polygon", coordinates=[[
                                            [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                        ]]),
                                        agg_methods=['mean', 'count'],
                                        max_valids=-1)
        expected_result = [{'count': 122392,
                            'count_tot': 159600,
                            'mean': 56.12519223634024,
                            'time': '2017-01-16T10:09:22Z'},
                           {'count': 132066,
                            'count_tot': 159600,
                            'mean': 49.70755256053988,
                            'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_collection(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx,
                                        'demo', 'conc_tsm',
                                        dict(type="GeometryCollection",
                                             geometries=[
                                                 dict(type="Point", coordinates=[2.1, 51.4])]),
                                        agg_methods=['mean', 'count'],
                                        start_date=np.datetime64('2017-01-15'),
                                        end_date=np.datetime64('2017-01-29'))
        expected_result = [[{'mean': 3.534773588180542, 'time': '2017-01-16T10:09:22Z'},
                            {'mean': None, 'time': '2017-01-25T09:35:51Z'},
                            {'mean': None, 'time': '2017-01-26T10:50:17Z'},
                            {'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon_collection(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx,
                                        'demo', 'conc_tsm',
                                        dict(type="GeometryCollection",
                                             geometries=[dict(type="Polygon", coordinates=[[
                                                 [1., 51.], [2., 51.], [2., 52.], [1., 52.],
                                                 [1., 51.]
                                             ]])]),
                                        agg_methods=['mean', 'count'])
        expected_result = [[{'count': 122392,
                             'count_tot': 159600,
                             'mean': 56.12519223634024,
                             'time': '2017-01-16T10:09:22Z'},
                            {'count': 0,
                             'count_tot': 159600,
                             'mean': None,
                             'time': '2017-01-25T09:35:51Z'},
                            {'count': 0,
                             'count_tot': 159600,
                             'mean': None,
                             'time': '2017-01-26T10:50:17Z'},
                            {'count': 132066,
                             'count_tot': 159600,
                             'mean': 49.70755256053988,
                             'time': '2017-01-28T09:58:11Z'},
                            {'count': 0,
                             'count_tot': 159600,
                             'mean': None,
                             'time': '2017-01-30T10:46:34Z'}]]

        self.assertAlmostEqualDeep(expected_result, actual_result)


class CollectTimeSeriesResultTest(unittest.TestCase, AlmostEqualDeepMixin):

    def test_all_types_converted_correctly(self):
        self.maxDiff = None

        time_series_ds = xr.Dataset(
            data_vars=dict(time=xr.DataArray(pd.date_range(start='2010-04-05', periods=4, freq='1D'), dims='time'),
                           a=xr.DataArray([True, False, False, True], dims='time'),
                           b=xr.DataArray([32, 33, 35, 34], dims='time'),
                           c=xr.DataArray([0.4, 0.2, np.nan, 0.7], dims='time')))
        result = _collect_timeseries_result(time_series_ds, {'a': 'a', 'b': 'b', 'c': 'c'}, max_valids=None)
        self.assertEqual([{'a': True, 'b': 32, 'c': 0.4, 'time': '2010-04-05T00:00:00Z'},
                          {'a': False, 'b': 33, 'c': 0.2, 'time': '2010-04-06T00:00:00Z'},
                          {'a': False, 'b': 35, 'c': None, 'time': '2010-04-07T00:00:00Z'},
                          {'a': True, 'b': 34, 'c': 0.7, 'time': '2010-04-08T00:00:00Z'}],
                         result)
        self.assertEqual('[{"a": true, "b": 32, "c": 0.4, "time": "2010-04-05T00:00:00Z"},'
                         ' {"a": false, "b": 33, "c": 0.2, "time": "2010-04-06T00:00:00Z"},'
                         ' {"a": false, "b": 35, "c": null, "time": "2010-04-07T00:00:00Z"},'
                         ' {"a": true, "b": 34, "c": 0.7, "time": "2010-04-08T00:00:00Z"}]',
                         json.dumps(result))

    def test_num_valids(self):
        self.maxDiff = None

        time_series_ds = xr.Dataset(
            data_vars=dict(time=xr.DataArray(pd.date_range(start='2010-04-05', periods=5, freq='1D'), dims='time'),
                           a=xr.DataArray([np.nan, 5, np.nan, 2, 3], dims='time'),
                           b=xr.DataArray([np.nan, 33, np.nan, np.nan, 23], dims='time'),
                           c=xr.DataArray([np.nan, 0.2, np.nan, 0.7, np.nan], dims='time')))

        result = _collect_timeseries_result(time_series_ds, {'a': 'a', 'b': 'b', 'c': 'c'}, max_valids=-1)
        self.assertEqual([{'a': 5.0, 'b': 33.0, 'c': 0.2, 'time': '2010-04-06T00:00:00Z'},
                          {'a': 2.0, 'b': None, 'c': 0.7, 'time': '2010-04-08T00:00:00Z'},
                          {'a': 3.0, 'b': 23.0, 'c': None, 'time': '2010-04-09T00:00:00Z'}],
                         result)

        result = _collect_timeseries_result(time_series_ds, {'a': 'a', 'b': 'b', 'c': 'c'}, max_valids=2)
        self.assertEqual([{'a': 2.0, 'b': None, 'c': 0.7, 'time': '2010-04-08T00:00:00Z'},
                          {'a': 3.0, 'b': 23.0, 'c': None, 'time': '2010-04-09T00:00:00Z'}],
                         result)

    def test_count(self):
        self.maxDiff = None

        time_series_ds = xr.Dataset(
            data_vars=dict(time=xr.DataArray(pd.date_range(start='2010-04-05', periods=3, freq='1D'), dims='time'),
                           c=xr.DataArray([23, 78, 74], dims='time')),
            attrs=dict(max_number_of_observations=82))

        result = _collect_timeseries_result(time_series_ds, {'count': 'c'}, max_valids=-1)
        self.assertEqual([{'count': 23, 'count_tot': 82, 'time': '2010-04-05T00:00:00Z'},
                          {'count': 78, 'count_tot': 82, 'time': '2010-04-06T00:00:00Z'},
                          {'count': 74, 'count_tot': 82, 'time': '2010-04-07T00:00:00Z'}],
                         result)


@unittest.skipUnless(os.environ.get('XCUBE_TS_PERF_TEST') == '1', 'XCUBE_TS_PERF_TEST is not 1')
class TsPerfTest(unittest.TestCase):

    def test_point_ts_perf(self):
        TEST_CUBE = 'ts_test.zarr'

        if not os.path.isdir(TEST_CUBE):
            from xcube.core.new import new_cube
            cube = new_cube(time_periods=2000, variables=dict(analysed_sst=280.4))
            cube = cube.chunk(dict(time=1, lon=90, lat=90))
            cube.to_zarr(TEST_CUBE)

        ctx = ServiceContext(
            base_dir='.',
            config=dict(
                Datasets=[
                    dict(Identifier='ts_test',
                         FileSystem='file',
                         Path=TEST_CUBE,
                         Format='zarr')
                ]
            ))

        N = 5
        import random
        import time
        time_sum = 0.0
        for i in range(N):
            lon = -180 + 360 * random.random()
            lat = -90 + 180 * random.random()

            t1 = time.perf_counter()
            result = get_time_series(ctx, 'ts_test', 'analysed_sst', dict(type='Point', coordinates=[lon, lat]))
            t2 = time.perf_counter()

            self.assertIsInstance(result, list)
            self.assertEqual(2000, len(result))

            time_delta = t2 - t1
            time_sum += time_delta
            print(f'test {i + 1} took {time_delta} seconds')

        print(f'all tests took {time_sum / N} seconds in average')


# test/webapi/controllers/test_ts_legacy.py

import unittest

import numpy as np

from test.webapi.helpers import new_test_service_context
from xcube.webapi.controllers.ts_legacy import get_time_series_info, get_time_series_for_point, \
    get_time_series_for_geometry, get_time_series_for_geometry_collection
from test.mixins import AlmostEqualDeepMixin


class TsLegacyControllerTest(unittest.TestCase, AlmostEqualDeepMixin):

    def setUp(self) -> None:
        self.maxDiff = None

    def test_get_time_series_for_point_invalid_lat_and_lon(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo', 'conc_tsm',
                                                  lon=-150.0, lat=-30.0)
        expected_result = {'results': []}
        self.assertEqual(expected_result, actual_result)

    def test_get_time_series_for_point(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo', 'conc_tsm',
                                                  lon=2.1, lat=51.4,
                                                  start_date=np.datetime64('2017-01-15'),
                                                  end_date=np.datetime64('2017-01-29'))
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 3.534773588180542,
                                                   'totalCount': 1,
                                                   'validCount': 1}},
                                       {'date': '2017-01-25T09:35:51Z',
                                        'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                       {'date': '2017-01-26T10:50:17Z',
                                        'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_one_valid(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo', 'conc_tsm',
                                                  lon=2.1, lat=51.4,
                                                  start_date=np.datetime64('2017-01-15'),
                                                  end_date=np.datetime64('2017-01-29'),
                                                  max_valids=1)
        expected_result = {'results': [{'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_only_valids(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo', 'conc_tsm',
                                                  lon=2.1, lat=51.4,
                                                  start_date=np.datetime64('2017-01-15'),
                                                  end_date=np.datetime64('2017-01-29'),
                                                  max_valids=-1)
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 3.534773588180542,
                                                   'totalCount': 1,
                                                   'validCount': 1}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_with_uncertainty(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo-1w', 'conc_tsm',
                                                  lon=2.1, lat=51.4,
                                                  start_date=np.datetime64('2017-01-15'),
                                                  end_date=np.datetime64('2017-01-29'))
        expected_result = {'results': [{'date': '2017-01-22T00:00:00Z',
                                        'result': {'average': 3.534773588180542,
                                                   'totalCount': 1,
                                                   'validCount': 1}},
                                       {'date': '2017-01-29T00:00:00Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_point(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Point", coordinates=[2.1, 51.4]),
                                                     start_date=np.datetime64('2017-01-15'),
                                                     end_date=np.datetime64('2017-01-29'),
                                                     include_count=True)
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 3.534773588180542,
                                                   'totalCount': 1,
                                                   'validCount': 1}},
                                       {'date': '2017-01-25T09:35:51Z',
                                        'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                       {'date': '2017-01-26T10:50:17Z',
                                        'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_polygon(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Polygon", coordinates=[[
                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                                     ]]),
                                                     include_count=True)
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 56.12519223634024,
                                                   'totalCount': 159600,
                                                   'validCount': 122392}},
                                       {'date': '2017-01-25T09:35:51Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'validCount': 0}},
                                       {'date': '2017-01-26T10:50:17Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'validCount': 0}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 49.70755256053988,
                                                   'totalCount': 159600,
                                                   'validCount': 132066}},
                                       {'date': '2017-01-30T10:46:34Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'validCount': 0}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_polygon_with_stdev(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Polygon", coordinates=[[
                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                                     ]]),
                                                     include_count=True,
                                                     include_stdev=True)

        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 56.12519223634024,
                                                   'totalCount': 159600,
                                                   'uncertainty': 40.78859862094861,
                                                   'validCount': 122392}},
                                       {'date': '2017-01-25T09:35:51Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'uncertainty': None,
                                                   'validCount': 0}},
                                       {'date': '2017-01-26T10:50:17Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'uncertainty': None,
                                                   'validCount': 0}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 49.70755256053988,
                                                   'totalCount': 159600,
                                                   'uncertainty': 34.98868194514786,
                                                   'validCount': 132066}},
                                       {'date': '2017-01-30T10:46:34Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'uncertainty': None,
                                                   'validCount': 0}}]}

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_polygon_one_valid(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Polygon", coordinates=[[
                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                                     ]]),
                                                     include_count=True,
                                                     max_valids=1)
        expected_result = {'results': [{'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 49.70755256053988,
                                                   'totalCount': 159600,
                                                   'validCount': 132066}}]}

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_polygon_only_valids(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Polygon", coordinates=[[
                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                                     ]]),
                                                     include_count=True,
                                                     max_valids=-1)
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 56.12519223634024,
                                                   'totalCount': 159600,
                                                   'validCount': 122392}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 49.70755256053988,
                                                   'totalCount': 159600,
                                                   'validCount': 132066}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometries_incl_point(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry_collection(ctx,
                                                                'demo', 'conc_tsm',
                                                                dict(type="GeometryCollection",
                                                                     geometries=[
                                                                         dict(type="Point", coordinates=[2.1, 51.4])]),
                                                                start_date=np.datetime64('2017-01-15'),
                                                                end_date=np.datetime64('2017-01-29'),
                                                                include_count=True)
        expected_result = {'results': [[{'date': '2017-01-16T10:09:22Z',
                                         'result': {'average': 3.534773588180542,
                                                    'totalCount': 1,
                                                    'validCount': 1}},
                                        {'date': '2017-01-25T09:35:51Z',
                                         'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                        {'date': '2017-01-26T10:50:17Z',
                                         'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                        {'date': '2017-01-28T09:58:11Z',
                                         'result': {'average': 20.12085723876953,
                                                    'totalCount': 1,
                                                    'validCount': 1}}]]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometries_incl_polygon(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry_collection(ctx,
                                                                'demo', 'conc_tsm',
                                                                dict(type="GeometryCollection",
                                                                     geometries=[dict(type="Polygon", coordinates=[[
                                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.],
                                                                         [1., 51.]
                                                                     ]])]),
                                                                include_count=True)
        expected_result = {'results': [[{'date': '2017-01-16T10:09:22Z',
                                         'result': {'average': 56.12519223634024,
                                                    'totalCount': 159600,
                                                    'validCount': 122392}},
                                        {'date': '2017-01-25T09:35:51Z',
                                         'result': {'average': None,
                                                    'totalCount': 159600,
                                                    'validCount': 0}},
                                        {'date': '2017-01-26T10:50:17Z',
                                         'result': {'average': None,
                                                    'totalCount': 159600,
                                                    'validCount': 0}},
                                        {'date': '2017-01-28T09:58:11Z',
                                         'result': {'average': 49.70755256053988,
                                                    'totalCount': 159600,
                                                    'validCount': 132066}},
                                        {'date': '2017-01-30T10:46:34Z',
                                         'result': {'average': None,
                                                    'totalCount': 159600,
                                                    'validCount': 0}}]]}

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_info(self):
        self.maxDiff = None

        ctx = new_test_service_context()
        info = get_time_series_info(ctx)

        expected_result = self._get_expected_info_dict()
        self.assertEqual(expected_result, info)

    @staticmethod
    def _get_expected_info_dict():
        expected_dict = {'layers': []}
        bounds = {'xmin': 0.0, 'ymin': 50.0,
                  'xmax': 5.0, 'ymax': 52.5}
        demo_times = ['2017-01-16T10:09:22Z',
                      '2017-01-25T09:35:51Z',
                      '2017-01-26T10:50:17Z',
                      '2017-01-28T09:58:11Z',
                      '2017-01-30T10:46:34Z']
        demo_variables = ['c2rcc_flags',
                          'conc_chl',
                          'conc_tsm',
                          'kd489',
                          'quality_flags']
        for demo_variable in demo_variables:
            dict_variable = {'name': f'demo.{demo_variable}', 'dates': demo_times, 'bounds': bounds}
            expected_dict['layers'].append(dict_variable)
        demo1w_times = ['2017-01-22T00:00:00Z', '2017-01-29T00:00:00Z', '2017-02-05T00:00:00Z']
        for demo_variable in demo_variables:
            dict_variable = {'name': f'demo-1w.{demo_variable}', 'dates': demo1w_times, 'bounds': bounds}
            expected_dict['layers'].append(dict_variable)
            dict_variable = {'name': f'demo-1w.{demo_variable}_stdev', 'dates': demo1w_times, 'bounds': bounds}
            expected_dict['layers'].append(dict_variable)
        return expected_dict


# test/webapi/controllers/ogc/test_wmts.py
#  The MIT License (MIT)
#  Copyright (c) 2022 by the xcube development team and contributors
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.

import os
import unittest

import pyproj

from test.webapi.helpers import get_res_test_dir
from test.webapi.helpers import new_test_service_context
from xcube.core.gridmapping import GridMapping
from xcube.webapi.context import ServiceContext
from xcube.webapi.controllers.ogc.wmts import (
    get_operations_metadata_element,
    get_service_identification_element,
    get_service_provider_element,
    get_tile_matrix_set_crs84_element,
    get_tile_matrix_set_web_mercator_element,
    get_wmts_capabilities_xml,
    get_crs84_bbox,
    WMTS_CRS84_TMS_ID,
    WMTS_WEB_MERCATOR_TMS_ID
)


class WmtsControllerTest(unittest.TestCase):

    def test_get_wmts_capabilities_xml_crs84(self):
        self.maxDiff = None
        with open(os.path.join(get_res_test_dir(),
                               'WMTSCapabilities-CRS84.xml')) as fp:
            expected_xml = fp.read()
        ctx = new_test_service_context()
        actual_xml = get_wmts_capabilities_xml(ctx,
                                               'http://bibo',
                                               WMTS_CRS84_TMS_ID)
        print(80 * '=')
        print(actual_xml)
        print(80 * '=')
        self.assertEqual(expected_xml, actual_xml)

    def test_get_wmts_capabilities_xml_web_mercator(self):
        self.maxDiff = None
        with open(os.path.join(get_res_test_dir(),
                               'WMTSCapabilities-OSM.xml')) as fp:
            expected_xml = fp.read()
        ctx = new_test_service_context()
        actual_xml = get_wmts_capabilities_xml(ctx,
                                               'http://bibo',
                                               WMTS_WEB_MERCATOR_TMS_ID)
        # print(80 * '=')
        # print(actual_xml)
        # print(80 * '=')
        self.assertEqual(expected_xml, actual_xml)


class WmtsControllerXmlGenTest(unittest.TestCase):
    ctx = ServiceContext('api/v1',
                         config=dict(
                             ServiceProvider=dict(
                                 ProviderName='Bibo',
                                 ServiceContact=dict(
                                     PositionName='Boss',
                                     ContactInfo=dict(
                                         Address=dict(
                                             City='NYC'
                                         ),
                                     )
                                 ),
                             )
                         ))

    def test_get_service_provider(self):
        element = get_service_provider_element(self.ctx)
        self.assertEqual(
            '<ows:ServiceProvider>\n'
            '  <ows:ProviderName>Bibo</ows:ProviderName>\n'
            '  <ows:ProviderSite xlink:href=""/>\n'
            '  <ows:ServiceContact>\n'
            '    <ows:IndividualName/>\n'
            '    <ows:PositionName>Boss</ows:PositionName>\n'
            '    <ows:ContactInfo>\n'
            '      <ows:Phone>\n'
            '        <ows:Voice/>\n'
            '        <ows:Facsimile/>\n'
            '      </ows:Phone>\n'
            '      <ows:Address>\n'
            '        <ows:DeliveryPoint/>\n'
            '        <ows:City>NYC</ows:City>\n'
            '        <ows:AdministrativeArea/>\n'
            '        <ows:PostalCode/>\n'
            '        <ows:Country/>\n'
            '        <ows:ElectronicMailAddress/>\n'
            '      </ows:Address>\n'
            '    </ows:ContactInfo>\n'
            '  </ows:ServiceContact>\n'
            '</ows:ServiceProvider>',
            element.to_xml(indent=2)
        )

    def test_get_operations_metadata(self):
        element = get_operations_metadata_element(self.ctx,
                                                  'https://bibo',
                                                  WMTS_WEB_MERCATOR_TMS_ID)
        self.assertEqual(
            '<ows:OperationsMetadata>\n'
            '  <ows:Operation name="GetCapabilities">\n'
            '    <ows:DCP>\n'
            '      <ows:HTTP>\n'
            '        <ows:Get xlink:href="https://bibo/api/v1/wmts/kvp?">\n'
            '          <ows:Constraint name="GetEncoding">\n'
            '            <ows:AllowedValues>\n'
            '              <ows:Value>KVP</ows:Value>\n'
            '            </ows:AllowedValues>\n'
            '          </ows:Constraint>\n'
            '        </ows:Get>\n'
            '        <ows:Get xlink:href="https://bibo/api/v1/wmts/'
            '1.0.0/WorldWebMercatorQuad/WMTSCapabilities.xml">\n'
            '          <ows:Constraint name="GetEncoding">\n'
            '            <ows:AllowedValues>\n'
            '              <ows:Value>REST</ows:Value>\n'
            '            </ows:AllowedValues>\n'
            '          </ows:Constraint>\n'
            '        </ows:Get>\n'
            '      </ows:HTTP>\n'
            '    </ows:DCP>\n'
            '  </ows:Operation>\n'
            '  <ows:Operation name="GetTile">\n'
            '    <ows:DCP>\n'
            '      <ows:HTTP>\n'
            '        <ows:Get xlink:href="https://bibo/api/v1/wmts/kvp?">\n'
            '          <ows:Constraint name="GetEncoding">\n'
            '            <ows:AllowedValues>\n'
            '              <ows:Value>KVP</ows:Value>\n'
            '            </ows:AllowedValues>\n'
            '          </ows:Constraint>\n'
            '        </ows:Get>\n'
            '        <ows:Get xlink:href="https://bibo/api/v1/wmts/1.0.0/">\n'
            '          <ows:Constraint name="GetEncoding">\n'
            '            <ows:AllowedValues>\n'
            '              <ows:Value>REST</ows:Value>\n'
            '            </ows:AllowedValues>\n'
            '          </ows:Constraint>\n'
            '        </ows:Get>\n'
            '      </ows:HTTP>\n'
            '    </ows:DCP>\n'
            '  </ows:Operation>\n'
            '</ows:OperationsMetadata>',
            element.to_xml(indent=2)
        )

    def test_get_tile_matrix_set_crs84_element(self):
        element = get_tile_matrix_set_crs84_element(num_levels=3)
        self.assertEqual(
            '<TileMatrixSet>\n'
            '  <ows:Identifier>WorldCRS84Quad</ows:Identifier>\n'
            '  <ows:Title>CRS84 for the World</ows:Title>\n'
            '  <ows:SupportedCRS>urn:ogc:def:crs:OGC:1.3:CRS84'
            '</ows:SupportedCRS>\n'
            '  <ows:BoundingBox crs="urn:ogc:def:crs:OGC:1.3:CRS84">\n'
            '    <ows:LowerCorner>-180 -90</ows:LowerCorner>\n'
            '    <ows:UpperCorner>180 90</ows:UpperCorner>\n'
            '  </ows:BoundingBox>\n'
            '  <WellKnownScaleSet>urn:ogc:def:wkss:OGC:1.0:GoogleCRS84Quad'
            '</WellKnownScaleSet>\n'
            '  <TileMatrix>\n'
            '    <ows:Identifier>0</ows:Identifier>\n'
            '    <ScaleDenominator>279541132.01435894</ScaleDenominator>\n'
            '    <TopLeftCorner>-180 90</TopLeftCorner>\n'
            '    <TileWidth>256</TileWidth>\n'
            '    <TileHeight>256</TileHeight>\n'
            '    <MatrixWidth>2</MatrixWidth>\n'
            '    <MatrixHeight>1</MatrixHeight>\n'
            '  </TileMatrix>\n'
            '  <TileMatrix>\n'
            '    <ows:Identifier>1</ows:Identifier>\n'
            '    <ScaleDenominator>139770566.00717947</ScaleDenominator>\n'
            '    <TopLeftCorner>-180 90</TopLeftCorner>\n'
            '    <TileWidth>256</TileWidth>\n'
            '    <TileHeight>256</TileHeight>\n'
            '    <MatrixWidth>4</MatrixWidth>\n'
            '    <MatrixHeight>2</MatrixHeight>\n'
            '  </TileMatrix>\n'
            '  <TileMatrix>\n'
            '    <ows:Identifier>2</ows:Identifier>\n'
            '    <ScaleDenominator>69885283.00358973</ScaleDenominator>\n'
            '    <TopLeftCorner>-180 90</TopLeftCorner>\n'
            '    <TileWidth>256</TileWidth>\n'
            '    <TileHeight>256</TileHeight>\n'
            '    <MatrixWidth>8</MatrixWidth>\n'
            '    <MatrixHeight>4</MatrixHeight>\n'
            '  </TileMatrix>\n'
            '</TileMatrixSet>',
            element.to_xml(indent=2)
        )

    def test_get_tile_matrix_set_web_mercator_element(self):
        element = get_tile_matrix_set_web_mercator_element(num_levels=3)
        self.assertEqual(
            ('<TileMatrixSet>\n'
             '  <ows:Identifier>WorldWebMercatorQuad</ows:Identifier>\n'
             '  <ows:Title>Google Maps Compatible for the World</ows:Title>\n'
             '  <ows:SupportedCRS>'
             'urn:ogc:def:crs:EPSG::3857</ows:SupportedCRS>\n'
             '  <ows:BoundingBox crs="urn:ogc:def:crs:EPSG::3857">\n'
             '    <ows:LowerCorner>'
             '-20037508.3427892 -20037508.3427892</ows:LowerCorner>\n'
             '    <ows:UpperCorner>'
             '20037508.3427892 20037508.3427892</ows:UpperCorner>\n'
             '  </ows:BoundingBox>\n'
             '  '
             '<WellKnownScaleSet>'
             'urn:ogc:def:wkss:OGC:1.0:GoogleMapsCompatible'
             '</WellKnownScaleSet>\n'
             '  <TileMatrix>\n'
             '    <ows:Identifier>0</ows:Identifier>\n'
             '    <ScaleDenominator>559082264.0287178</ScaleDenominator>\n'
             '    <TopLeftCorner>'
             '-20037508.3427892 20037508.3427892</TopLeftCorner>\n'
             '    <TileWidth>256</TileWidth>\n'
             '    <TileHeight>256</TileHeight>\n'
             '    <MatrixWidth>1</MatrixWidth>\n'
             '    <MatrixHeight>1</MatrixHeight>\n'
             '  </TileMatrix>\n'
             '  <TileMatrix>\n'
             '    <ows:Identifier>1</ows:Identifier>\n'
             '    <ScaleDenominator>279541132.0143589</ScaleDenominator>\n'
             '    <TopLeftCorner>'
             '-20037508.3427892 20037508.3427892</TopLeftCorner>\n'
             '    <TileWidth>256</TileWidth>\n'
             '    <TileHeight>256</TileHeight>\n'
             '    <MatrixWidth>2</MatrixWidth>\n'
             '    <MatrixHeight>2</MatrixHeight>\n'
             '  </TileMatrix>\n'
             '  <TileMatrix>\n'
             '    <ows:Identifier>2</ows:Identifier>\n'
             '    <ScaleDenominator>139770566.00717944</ScaleDenominator>\n'
             '    <TopLeftCorner>'
             '-20037508.3427892 20037508.3427892</TopLeftCorner>\n'
             '    <TileWidth>256</TileWidth>\n'
             '    <TileHeight>256</TileHeight>\n'
             '    <MatrixWidth>4</MatrixWidth>\n'
             '    <MatrixHeight>4</MatrixHeight>\n'
             '  </TileMatrix>\n'
             '</TileMatrixSet>'),
            element.to_xml(indent=2)
        )

    def test_service_identification_element(self):
        element = get_service_identification_element()
        self.assertEqual(
            (
                '<ows:ServiceIdentification>\n'
                '  <ows:Title>xcube WMTS</ows:Title>\n'
                '  <ows:Abstract>Web Map Tile Service (WMTS)'
                ' for xcube-conformant data cubes</ows:Abstract>\n'
                '  <ows:Keywords>\n'
                '    <ows:Keyword>tile</ows:Keyword>\n'
                '    <ows:Keyword>tile matrix set</ows:Keyword>\n'
                '    <ows:Keyword>map</ows:Keyword>\n'
                '  </ows:Keywords>\n'
                '  <ows:ServiceType>OGC WMTS</ows:ServiceType>\n'
                '  <ows:ServiceTypeVersion>1.0.0</ows:ServiceTypeVersion>\n'
                '  <ows:Fees>none</ows:Fees>\n'
                '  <ows:AccessConstraints>none</ows:AccessConstraints>\n'
                '</ows:ServiceIdentification>'
            ),
            element.to_xml(indent=2)
        )


class WmtsCrs84BboxTest(unittest.TestCase):
    def test_get_crs84_bbox_ok(self):
        t = pyproj.Transformer.from_crs('EPSG:4326', 'EPSG:3035',
                                        always_xy=True)
        p0 = t.transform(10, 50)

        gm = GridMapping.regular((100, 100),
                                 p0,
                                 (1000., 1000.),
                                 pyproj.CRS.from_string('EPSG:3035'))
        bbox = get_crs84_bbox(gm)
        self.assertIsInstance(bbox, tuple)
        self.assertEqual(4, len(bbox))
        self.assertAlmostEqual(10.0, bbox[0])
        self.assertAlmostEqual(49.991462404901604, bbox[1])
        self.assertAlmostEqual(11.421266448415976, bbox[2])
        self.assertAlmostEqual(50.8990377520989, bbox[3])

    def test_get_crs84_bbox_fail(self):
        # construct impossible GridMapping
        gm = GridMapping.regular((100, 100),
                                 (100000000, -1000000),
                                 (1000., 1000.),
                                 pyproj.CRS.from_string('EPSG:3035'))
        with self.assertRaises(ValueError):
            get_crs84_bbox(gm)
