    xy_coords = da.array(self.data, dtype=np.float64).rechunk((2, 3, 3))
        xy_bbox = compute_xy_bbox(xy_coords)
        self.assertEqual((10, 50, 17, 57), xy_bbox)

    def test_many_nans(self):
        w = 2000
        h = 1000
        x = np.full(h * w, np.nan)
        y = np.full(h * w, np.nan)

        x[np.random.randint(0, w)] = 73.
        y[np.random.randint(0, h)] = 34.

        xy_coords = da.array([x.reshape((h, w)), y.reshape((h, w))], dtype=np.float64) \
            .rechunk((2, 512, 512))
        xy_bbox = compute_xy_bbox(xy_coords)

        self.assertEqual((73., 34., 73., 34.), xy_bbox)


# test/core/gridmapping/test_cfconv.py

import shutil
import unittest
from typing import Set

import fsspec
import numpy as np
import pyproj
import xarray as xr

from test.s3test import MOTO_SERVER_ENDPOINT_URL
from test.s3test import S3Test
from xcube.core.gridmapping import GridMapping
from xcube.core.gridmapping.cfconv import GridCoords
from xcube.core.gridmapping.cfconv import GridMappingProxy
from xcube.core.gridmapping.cfconv import add_spatial_ref
from xcube.core.gridmapping.cfconv import get_dataset_grid_mapping_proxies
from xcube.core.new import new_cube
from xcube.core.store.fs.registry import new_fs_data_store

CRS_WGS84 = pyproj.crs.CRS(4326)
CRS_CRS84 = pyproj.crs.CRS.from_string("urn:ogc:def:crs:OGC:1.3:CRS84")
CRS_UTM_33N = pyproj.crs.CRS(32633)

CRS_ROTATED_POLE = pyproj.crs.CRS.from_cf(
    dict(grid_mapping_name="rotated_latitude_longitude",
         grid_north_pole_latitude=32.5,
         grid_north_pole_longitude=170.))


class GetDatasetGridMappingsTest(unittest.TestCase):
    def test_no_crs_lon_lat_common_names(self):
        dataset = xr.Dataset(coords=dict(lon=xr.DataArray(np.linspace(10, 12, 11), dims='lon'),
                                         lat=xr.DataArray(np.linspace(50, 52, 11), dims='lat')))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn(None, grid_mappings)
        grid_mapping = grid_mappings.get(None)
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual(CRS_WGS84, grid_mapping.crs)
        self.assertEqual('latitude_longitude', grid_mapping.name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('lon', grid_mapping.coords.x.name)
        self.assertEqual('lat', grid_mapping.coords.y.name)

    def test_no_crs_lon_lat_standard_names(self):
        dataset = xr.Dataset(coords=dict(weird_x=xr.DataArray(np.linspace(10, 12, 11), dims='i',
                                                              attrs=dict(standard_name='longitude')),
                                         weird_y=xr.DataArray(np.linspace(50, 52, 11), dims='j',
                                                              attrs=dict(standard_name='latitude'))))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn(None, grid_mappings)
        grid_mapping = grid_mappings.get(None)
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual(CRS_WGS84, grid_mapping.crs)
        self.assertEqual('latitude_longitude', grid_mapping.name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('weird_x', grid_mapping.coords.x.name)
        self.assertEqual('weird_y', grid_mapping.coords.y.name)

    def test_crs_x_y_with_common_names(self):
        dataset = xr.Dataset(dict(crs=xr.DataArray(0, attrs=CRS_UTM_33N.to_cf())),
                             coords=dict(x=xr.DataArray(np.linspace(1000, 12000, 11), dims='x'),
                                         y=xr.DataArray(np.linspace(5000, 52000, 11), dims='y')))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn('crs', grid_mappings)
        grid_mapping = grid_mappings.get('crs')
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual(CRS_UTM_33N, grid_mapping.crs)
        self.assertEqual('transverse_mercator', grid_mapping.name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('x', grid_mapping.coords.x.name)
        self.assertEqual('y', grid_mapping.coords.y.name)

    def test_crs_x_y_with_standard_names(self):
        dataset = xr.Dataset(dict(crs=xr.DataArray(0, attrs=CRS_UTM_33N.to_cf())),
                             coords=dict(myx=xr.DataArray(np.linspace(1000, 12000, 11), dims='x',
                                                          attrs=dict(standard_name='projection_x_coordinate')),
                                         myy=xr.DataArray(np.linspace(5000, 52000, 11), dims='y',
                                                          attrs=dict(standard_name='projection_y_coordinate'))))
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn('crs', grid_mappings)
        grid_mapping = grid_mappings.get('crs')
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual(CRS_UTM_33N, grid_mapping.crs)
        self.assertEqual('transverse_mercator', grid_mapping.name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('myx', grid_mapping.coords.x.name)
        self.assertEqual('myy', grid_mapping.coords.y.name)

    def test_latitude_longitude_with_x_y(self):
        # This is what we get when opening a CRS-84 GeoTIFF using rioxarray
        dataset = xr.Dataset(
            dict(
                band_1=xr.DataArray(np.zeros((11, 11)), dims=["y", "x"]),
                spatial_ref=xr.DataArray(0, attrs=CRS_CRS84.to_cf())
            ),
            coords=dict(
                x=xr.DataArray(np.linspace(10, 20, 11), dims='x'),
                y=xr.DataArray(np.linspace(50, 40, 11), dims='y'),
            )
        )
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn('spatial_ref', grid_mappings)
        grid_mapping = grid_mappings.get('spatial_ref')
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual(CRS_CRS84, grid_mapping.crs)
        self.assertEqual('latitude_longitude', grid_mapping.name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('x', grid_mapping.coords.x.name)
        self.assertEqual('y', grid_mapping.coords.y.name)

    def test_rotated_pole_with_common_names(self):
        dataset = xr.Dataset(
            dict(
                rotated_pole=xr.DataArray(0, attrs=CRS_ROTATED_POLE.to_cf())
            ),
            coords=dict(
                rlon=xr.DataArray(np.linspace(-180, 180, 11), dims='rlon'),
                rlat=xr.DataArray(np.linspace(0, 90, 11), dims='rlat')
            )
        )
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn('rotated_pole', grid_mappings)
        grid_mapping = grid_mappings.get('rotated_pole')
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual('Derived Geographic 2D CRS',
                         grid_mapping.crs.type_name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('rlon', grid_mapping.coords.x.name)
        self.assertEqual('rlat', grid_mapping.coords.y.name)

    def test_rotated_pole_with_standard_names(self):
        dataset = xr.Dataset(
            dict(
                rotated_pole=xr.DataArray(0,
                                          attrs=CRS_ROTATED_POLE.to_cf())
            ),
            coords=dict(
                u=xr.DataArray(np.linspace(-180, 180, 11), dims='u',
                               attrs=dict(standard_name='grid_longitude')),
                v=xr.DataArray(np.linspace(0, 90, 11), dims='v',
                               attrs=dict(standard_name='grid_latitude'))
            )
        )
        grid_mappings = get_dataset_grid_mapping_proxies(dataset)
        self.assertEqual(1, len(grid_mappings))
        self.assertIn('rotated_pole', grid_mappings)
        grid_mapping = grid_mappings.get('rotated_pole')
        self.assertIsInstance(grid_mapping, GridMappingProxy)
        self.assertEqual('Derived Geographic 2D CRS',
                         grid_mapping.crs.type_name)
        self.assertIsInstance(grid_mapping.coords, GridCoords)
        self.assertIsInstance(grid_mapping.coords.x, xr.DataArray)
        self.assertIsInstance(grid_mapping.coords.y, xr.DataArray)
        self.assertEqual('u', grid_mapping.coords.x.name)
        self.assertEqual('v', grid_mapping.coords.y.name)


class XarrayDecodeCfTest(unittest.TestCase):
    """Find out how xarray treats 1D and 2D coordinate variables when decode_cf=True or =False"""

    def test_cf_1d_coords(self):
        self._write_coords(*self._gen_1d())
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=True)
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=False)

    def test_cf_1d_data_vars(self):
        self._write_data_vars(*self._gen_1d())
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=True)
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=False)

    def test_cf_2d_coords(self):
        self._write_coords(*self._gen_2d())
        self.assertVarNames({'noise', 'crs'}, {'lon', 'lat'}, decode_cf=True)
        self.assertVarNames({'noise', 'crs', 'lon', 'lat'}, set(), decode_cf=False)

    def test_cf_2d_data_vars(self):
        self._write_data_vars(*self._gen_2d())
        self.assertVarNames({'noise', 'crs', 'lon', 'lat'}, set(), decode_cf=True)
        self.assertVarNames({'noise', 'crs', 'lon', 'lat'}, set(), decode_cf=False)

    def assertVarNames(self, exp_data_vars: Set[str], exp_coords: Set[str], decode_cf: bool):
        ds1 = xr.open_zarr('noise.zarr', decode_cf=decode_cf)
        self.assertEqual(exp_coords, set(ds1.coords))
        self.assertEqual(exp_data_vars, set(ds1.data_vars))

    @classmethod
    def _write_coords(cls, noise, crs, lon, lat):
        dataset = xr.Dataset(dict(noise=noise, crs=crs), coords=dict(lon=lon, lat=lat))
        dataset.to_zarr('noise.zarr', mode='w')

    @classmethod
    def _write_data_vars(cls, noise, crs, lon, lat):
        dataset = xr.Dataset(dict(noise=noise, crs=crs, lon=lon, lat=lat))
        dataset.to_zarr('noise.zarr', mode='w')

    @classmethod
    def tearDownClass(cls) -> None:
        shutil.rmtree('noise.zarr')

    def _gen_1d(self):
        noise = xr.DataArray(np.random.random((11, 11)), dims=('lat', 'lon'))
        crs = xr.DataArray(0, attrs=CRS_CRS84.to_cf())
        lon = xr.DataArray(np.linspace(10, 12, 11), dims='lon')
        lat = xr.DataArray(np.linspace(50, 52, 11), dims='lat')
        noise.attrs['grid_mapping'] = 'crs'
        lon.attrs['standard_name'] = 'longitude'
        lat.attrs['standard_name'] = 'latitude'
        self.assertEqual(('lon',), lon.dims)
        self.assertEqual(('lat',), lat.dims)
        return noise, crs, lon, lat

    def _gen_2d(self):
        noise = xr.DataArray(np.random.random((11, 11)), dims=('y', 'x'))
        crs = xr.DataArray(0, attrs=CRS_CRS84.to_cf())
        lon = xr.DataArray(np.linspace(10, 12, 11), dims='x')
        lat = xr.DataArray(np.linspace(50, 52, 11), dims='y')
        lat, lon = xr.broadcast(lat, lon)
        noise.attrs['grid_mapping'] = 'crs'
        lon.attrs['standard_name'] = 'longitude'
        lat.attrs['standard_name'] = 'latitude'
        self.assertEqual(('y', 'x'), lon.dims)
        self.assertEqual(('y', 'x'), lat.dims)
        return noise, crs, lon, lat


class AddSpatialRefTest(S3Test, unittest.TestCase):

    def test_add_spatial_ref(self):
        self.assert_add_spatial_ref_ok(None, None)
        self.assert_add_spatial_ref_ok(None, ('cx', 'cy'))
        self.assert_add_spatial_ref_ok('crs', None)
        self.assert_add_spatial_ref_ok('crs', ('cx', 'cy'))

    def assert_add_spatial_ref_ok(self, crs_var_name, xy_dim_names):

        root = 'eurodatacube-test/xcube-eea'
        data_id = 'test.zarr'
        crs = pyproj.CRS.from_string("EPSG:3035")

        if xy_dim_names:
            x_name, y_name = xy_dim_names
        else:
            x_name, y_name = 'x', 'y'

        cube = new_cube(x_name=x_name,
                        y_name=y_name,
                        x_start=0,
                        y_start=0,
                        x_res=10,
                        y_res=10,
                        x_units='metres',
                        y_units='metres',
                        drop_bounds=True,
                        width=100,
                        height=100,
                        variables=dict(A=1.3, B=8.3))

        storage_options = dict(
            anon=False,
            client_kwargs=dict(
                endpoint_url=MOTO_SERVER_ENDPOINT_URL,
            )
        )

        fs: fsspec.AbstractFileSystem = fsspec.filesystem('s3',
                                                          **storage_options)
        if fs.isdir(root):
            fs.rm(root, recursive=True)
        fs.mkdirs(root, exist_ok=True)

        data_store = new_fs_data_store('s3',
                                       root=root,
                                       storage_options=storage_options)

        data_store.write_data(cube, data_id=data_id)
        cube = data_store.open_data(data_id)
        self.assertEqual({'A', 'B', 'time', x_name, y_name},
                         set(cube.variables))

        with self.assertRaises(ValueError) as cm:
            GridMapping.from_dataset(cube)
        self.assertEqual(
            ('cannot find any grid mapping in dataset',),
            cm.exception.args
        )

        path = f"{root}/{data_id}"
        group_store = fs.get_mapper(path, create=True)

        expected_crs_var_name = crs_var_name or 'spatial_ref'

        self.assertTrue(fs.exists(path))
        self.assertFalse(fs.exists(f"{path}/{expected_crs_var_name}"))
        self.assertFalse(fs.exists(f"{path}/{expected_crs_var_name}/.zarray"))
        self.assertFalse(fs.exists(f"{path}/{expected_crs_var_name}/.zattrs"))

        kwargs = {}
        if crs_var_name is not None:
            kwargs.update(crs_var_name=crs_var_name)
        if xy_dim_names is not None:
            kwargs.update(xy_dim_names=xy_dim_names)
        add_spatial_ref(group_store, crs, **kwargs)

        self.assertTrue(fs.exists(f"{path}/{expected_crs_var_name}"))
        self.assertTrue(fs.exists(f"{path}/{expected_crs_var_name}/.zarray"))
        self.assertTrue(fs.exists(f"{path}/{expected_crs_var_name}/.zattrs"))

        cube = data_store.open_data(data_id)
        self.assertEqual({'A', 'B', 'time',
                          x_name, y_name, expected_crs_var_name},
                         set(cube.variables))
        self.assertEqual(expected_crs_var_name,
                         cube.A.attrs.get('grid_mapping'))
        self.assertEqual(expected_crs_var_name,
                         cube.B.attrs.get('grid_mapping'))

        gm = GridMapping.from_dataset(cube)
        self.assertIn("LAEA Europe", gm.crs.srs)


# test/core/gridmapping/test_coords.py

import unittest

import dask.array as da
import numpy as np
import pyproj
import xarray as xr

from xcube.core.gridmapping import GridMapping

# noinspection PyProtectedMember

GEO_CRS = pyproj.crs.CRS(4326)
NOT_A_GEO_CRS = pyproj.crs.CRS(5243)


# noinspection PyMethodMayBeStatic
class Coords1DGridMappingTest(unittest.TestCase):

    def test_1d_j_axis_down(self):
        x_coords = xr.DataArray(np.linspace(1.5, 8.5, 8), dims='lon')
        y_coords = xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat')
        gm = GridMapping.from_coords(x_coords=x_coords,
                                     y_coords=y_coords,
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((1, -5, 9, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)
        self.assertTrue(hasattr(gm, 'x_coords'))
        self.assertTrue(hasattr(gm, 'y_coords'))
        # noinspection PyUnresolvedReferences
        self.assertIs(x_coords, gm.x_coords)
        # noinspection PyUnresolvedReferences
        self.assertIs(y_coords, gm.y_coords)

    def test_1d_j_axis_up(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(np.linspace(1.5, 8.5, 8), dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(-4.5, 4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((1, -5, 9, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)

    def test_1d_lon_360(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(np.linspace(177.5, 184.5, 8), dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((177, -5, 185, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_1d_anti_meridian(self):
        lon = np.linspace(177.5, 184.5, 8)
        lon = np.where(lon > 180, lon - 360, lon)
        gm = GridMapping.from_coords(x_coords=xr.DataArray(lon, dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((177, -5, 185, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_1d_tiles_given(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(np.linspace(177.5, 184.5, 8), dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS,
                                     tile_size=(5, 3))
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((5, 3), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((177, -5, 185, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_1d_tiles_from_coords_chunks(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(da.linspace(177.5, 184.5, 8, chunks=4), dims='lon'),
                                     y_coords=xr.DataArray(da.linspace(4.5, -4.5, 10, chunks=5), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((4, 5), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((177, -5, 185, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_1d_x_irregular(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray([1.5, 2.5, 3.5, 4.5, 5.49, 6.5, 7.5, 8.5], dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        self.assertEqual((8, 10), gm.size)
        self.assertEqual((8, 10), gm.tile_size)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual((1, -5, 9, 5), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)

    def test_1d_xy_coords(self):
        gm = GridMapping.from_coords(x_coords=xr.DataArray(np.linspace(1.5, 8.5, 8), dims='lon'),
                                     y_coords=xr.DataArray(np.linspace(4.5, -4.5, 10), dims='lat'),
                                     crs=GEO_CRS)
        xy_coords = gm.xy_coords
        self.assertIsInstance(xy_coords, xr.DataArray)
        self.assertIs(xy_coords, gm.xy_coords)
        self.assertEqual(('coord', 'lat', 'lon'), xy_coords.dims)
        self.assertEqual((2, 10, 8), xy_coords.shape)
        self.assertEqual(('lon', 'lat'), gm.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm.xy_dim_names)

    def test_to_coords(self):
        gm = GridMapping.regular(size=(10, 6),
                                 xy_min=(-2600.0, 1200.0),
                                 xy_res=10.0,
                                 crs=NOT_A_GEO_CRS)
        cv = gm.to_coords(reuse_coords=False)
        self.assertIn("x", cv)
        self.assertIn("y", cv)
        self.assertEqual(np.float64, cv["x"].dtype)
        self.assertEqual(np.float64, cv["y"].dtype)

        gm2 = GridMapping.from_coords(cv['x'].astype(np.float32),
                                      cv['y'].astype(np.float32),
                                      gm.crs)
        cv2 = gm2.to_coords(xy_var_names=("a", "b"),
                            xy_dim_names=("u", "v"),
                            reuse_coords=True)
        self.assertIn("a", cv2)
        self.assertIn("b", cv2)
        self.assertEqual(np.float32, cv2["a"].dtype)
        self.assertEqual(np.float32, cv2["b"].dtype)


class Coords2DGridMappingTest(unittest.TestCase):

    def test_2d(self):
        x_coords = xr.DataArray([[10.0, 10.1, 10.2, 10.3], [10.1, 10.2, 10.3, 10.4], [10.2, 10.3, 10.4, 10.5], ],
                                dims=('lat', 'lon'))
        y_coords = xr.DataArray([[52.0, 52.2, 52.4, 52.6], [52.2, 52.4, 52.6, 52.8], [52.4, 52.6, 52.8, 53.0], ],
                                dims=('lat', 'lon'))
        gm = GridMapping.from_coords(
            x_coords=x_coords,
            y_coords=y_coords,
            crs=GEO_CRS)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((4, 3), gm.tile_size)
        self.assertEqual((0.3, 0.3), gm.xy_res)
        self.assertEqual((9.85, 51.85, 10.65, 53.15), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)
        self.assertTrue(hasattr(gm, 'x_coords'))
        self.assertTrue(hasattr(gm, 'y_coords'))
        # noinspection PyUnresolvedReferences
        self.assertIs(x_coords, gm.x_coords)
        # noinspection PyUnresolvedReferences
        self.assertIs(y_coords, gm.y_coords)

    def test_2d_tile_size_from_chunks(self):
        gm = GridMapping.from_coords(
            x_coords=xr.DataArray(da.array([
                [10.0, 10.1, 10.2, 10.3],
                [10.1, 10.2, 10.3, 10.4],
                [10.2, 10.3, 10.4, 10.5],
            ]).rechunk((2, 3)), dims=('lat', 'lon')),
            y_coords=xr.DataArray(da.array([
                [52.0, 52.2, 52.4, 52.6],
                [52.2, 52.4, 52.6, 52.8],
                [52.4, 52.6, 52.8, 53.0],
            ]).rechunk((2, 3)), dims=('lat', 'lon')),
            crs=GEO_CRS)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((3, 2), gm.tile_size)

    def test_2d_regular(self):
        gm = GridMapping.from_coords(
            x_coords=xr.DataArray([
                [10.2, 10.3, 10.4, 10.5],
                [10.2, 10.3, 10.4, 10.5],
                [10.2, 10.3, 10.4, 10.5],
            ], dims=('lat', 'lon')),
            y_coords=xr.DataArray([
                [52.4, 52.4, 52.4, 52.4],
                [52.6, 52.6, 52.6, 52.6],
                [52.8, 52.8, 52.8, 52.8],
            ], dims=('lat', 'lon')),
            crs=GEO_CRS)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((4, 3), gm.tile_size)
        self.assertAlmostEqual(0.1, gm.x_res)
        self.assertAlmostEqual(0.2, gm.y_res)
        self.assertAlmostEqual(10.15, gm.x_min)
        self.assertAlmostEqual(52.3, gm.y_min)
        self.assertAlmostEqual(10.55, gm.x_max)
        self.assertAlmostEqual(52.9, gm.y_max)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual(False, gm.is_lon_360)

    def test_2d_anti_meridian(self):
        gm = GridMapping.from_coords(
            x_coords=xr.DataArray([
                [+177.5, +178.5, +179.5, -179.5],
                [+178.5, +179.5, -179.5, -178.5],
                [+179.5, -179.5, -178.5, -177.5],
            ], dims=('lat', 'lon')),
            y_coords=xr.DataArray([
                [52.4, 52.4, 52.4, 52.4],
                [52.6, 52.6, 52.6, 52.6],
                [52.8, 52.8, 52.8, 52.8],
            ], dims=('lat', 'lon')),
            crs=GEO_CRS)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((4, 3), gm.tile_size)
        self.assertAlmostEqual(0.2, gm.x_res)
        self.assertAlmostEqual(0.2, gm.y_res)
        self.assertEqual((177.4, 52.3, 182.6, 52.9), gm.xy_bbox)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual(True, gm.is_lon_360)

    def test_to_regular(self):
        lon = xr.DataArray([[1.0, 6.0],
                            [0.0, 2.0]], dims=('y', 'x'))
        lat = xr.DataArray([[56.0, 53.0],
                            [52.0, 50.0]], dims=('y', 'x'))

        gm_irr = GridMapping.from_coords(lon, lat, GEO_CRS)
        gm_reg_actual = gm_irr.to_regular()
        gm_reg_expected = GridMapping.regular(size=(4, 4),
                                              xy_min=(-2, 48),
                                              xy_res=4.0,
                                              crs=GEO_CRS)
        self.assertEqual(gm_reg_expected.size, gm_reg_actual.size)
        self.assertEqual(gm_reg_expected.tile_size, gm_reg_actual.tile_size)
        self.assertEqual(gm_reg_expected.xy_res, gm_reg_actual.xy_res)
        self.assertEqual(gm_reg_expected.xy_bbox, gm_reg_actual.xy_bbox)
        self.assertEqual(gm_reg_expected.crs, gm_reg_actual.crs)

    def test_2d_xy_coords(self):
        gm = GridMapping.from_coords(
            x_coords=xr.DataArray([
                [10.0, 10.1, 10.2, 10.3],
                [10.1, 10.2, 10.3, 10.4],
                [10.2, 10.3, 10.4, 10.5],
            ], dims=('lat', 'lon')),
            y_coords=xr.DataArray([
                [52.0, 52.2, 52.4, 52.6],
                [52.2, 52.4, 52.6, 52.8],
                [52.4, 52.6, 52.8, 53.0],
            ], dims=('lat', 'lon')),
            crs=GEO_CRS)
        xy_coords = gm.xy_coords
        self.assertIsInstance(xy_coords, xr.DataArray)
        self.assertIs(xy_coords, gm.xy_coords)
        self.assertEqual(('coord', 'lat', 'lon'), xy_coords.dims)
        self.assertEqual((2, 3, 4), xy_coords.shape)
        self.assertEqual(('lon', 'lat'), gm.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm.xy_dim_names)


# test/core/gridmapping/test_dataset.py

import os.path
import unittest

import numpy as np
import pyproj
import xarray as xr

import xcube.core.new
from test.sampledata import create_s2plus_dataset
from xcube.core.gridmapping import GridMapping

# noinspection PyProtectedMember

GEO_CRS = pyproj.crs.CRS(4326)
NOT_A_GEO_CRS = pyproj.crs.CRS(5243)


# noinspection PyMethodMayBeStatic
class DatasetGridMappingTest(unittest.TestCase):
    def test_from_regular_cube(self):
        dataset = xcube.core.new.new_cube(variables=dict(rad=0.5))
        gm = GridMapping.from_dataset(dataset)
        self.assertEqual((360, 180), gm.size)
        self.assertEqual((360, 180), gm.tile_size)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual((1, 1), gm.xy_res)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_lon_360)
        self.assertEqual(True, gm.is_j_axis_up)
        self.assertEqual((2, 180, 360), gm.xy_coords.shape)
        self.assertEqual(('coord', 'lat', 'lon'), gm.xy_coords.dims)

    def test_from_regular_cube_with_crs(self):
        dataset = xcube.core.new.new_cube(variables=dict(rad=0.5),
                                          x_start=0,
                                          y_start=0,
                                          x_name='x',
                                          y_name='y',
                                          crs='epsg:25832')
        gm1 = GridMapping.from_dataset(dataset)
        self.assertEqual(pyproj.CRS.from_string('epsg:25832'), gm1.crs)
        dataset = dataset.drop_vars('crs')
        gm2 = GridMapping.from_dataset(dataset)
        self.assertEqual(GEO_CRS, gm2.crs)
        gm3 = GridMapping.from_dataset(dataset, crs=gm1.crs)
        self.assertEqual(gm1.crs, gm3.crs)
        self.assertEqual(('x', 'y'), gm3.xy_var_names)
        self.assertEqual(('x', 'y'), gm3.xy_dim_names)

    def test_from_regular_cube_no_chunks_and_chunks(self):
        dataset = xcube.core.new.new_cube(variables=dict(rad=0.5))
        gm1 = GridMapping.from_dataset(dataset)
        self.assertEqual((360, 180), gm1.tile_size)
        dataset = dataset.chunk(dict(lon=10, lat=20))
        gm2 = GridMapping.from_dataset(dataset)
        self.assertEqual((10, 20), gm2.tile_size)

    def test_from_non_regular_cube(self):
        lon = np.array([[8, 9.3, 10.6, 11.9],
                        [8, 9.2, 10.4, 11.6],
                        [8, 9.1, 10.2, 11.3]], dtype=np.float32)
        lat = np.array([[56, 56.1, 56.2, 56.3],
                        [55, 55.2, 55.4, 55.6],
                        [54, 54.3, 54.6, 54.9]], dtype=np.float32)
        rad = np.random.random(3 * 4).reshape((3, 4))
        dims = ('y', 'x')
        dataset = xr.Dataset(dict(lon=xr.DataArray(lon, dims=dims),
                                  lat=xr.DataArray(lat, dims=dims),
                                  rad=xr.DataArray(rad, dims=dims)))
        gm = GridMapping.from_dataset(dataset)
        self.assertEqual((4, 3), gm.size)
        self.assertEqual((4, 3), gm.tile_size)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(False, gm.is_lon_360)
        self.assertEqual(None, gm.is_j_axis_up)
        self.assertEqual((2, 3, 4), gm.xy_coords.shape)
        self.assertEqual(('coord', 'y', 'x'), gm.xy_coords.dims)
        self.assertEqual((0.8, 0.8), gm.xy_res)

    def test_from_real_olci(self):
        olci_l2_path = os.path.join(os.path.dirname(__file__),
                                    '..', '..', '..',
                                    'examples', 'notebooks', 'inputdata',
                                    'S3-OLCI-L2A.zarr.zip')

        dataset = xr.open_zarr(olci_l2_path)
        gm = GridMapping.from_dataset(dataset)
        self.assertEqual((1189, 1890), gm.size)
        self.assertEqual((512, 512), gm.tile_size)
        self.assertEqual(GEO_CRS, gm.crs)
        self.assertEqual((0.0025, 0.0025), gm.xy_res)
        # self.assertAlmostEqual(12.693771178309552, gm.x_min)
        # self.assertAlmostEqual(20.005413821690446, gm.x_max)
        # self.assertAlmostEqual(55.19965017830955, gm.y_min)
        # self.assertAlmostEqual(60.63871982169044, gm.y_max)
        self.assertEqual(False, gm.is_regular)
        self.assertEqual(False, gm.is_lon_360)
        self.assertEqual(None, gm.is_j_axis_up)
        self.assertEqual((2, 1890, 1189), gm.xy_coords.shape)
        self.assertEqual(('coord', 'y', 'x'), gm.xy_coords.dims)

        gm = gm.to_regular()
        self.assertEqual((2926, 2177), gm.size)

    def test_from_sentinel_2(self):
        dataset = create_s2plus_dataset()

        gm = GridMapping.from_dataset(dataset)
        # Should pick the projected one which is regular
        self.assertEqual("Derived Projected CRS", gm.crs.type_name)
        self.assertEqual(True, gm.is_regular)

        gm = GridMapping.from_dataset(dataset, prefer_is_regular=True)
        # Should pick the projected one which is regular
        self.assertEqual("Derived Projected CRS", gm.crs.type_name)
        self.assertEqual(True, gm.is_regular)

        gm = GridMapping.from_dataset(dataset, prefer_is_regular=False)
        # Should pick the geographic one which is irregular
        self.assertEqual('Geographic 2D CRS', gm.crs.type_name)
        self.assertEqual(False, gm.is_regular)

        gm = GridMapping.from_dataset(dataset, prefer_crs=GEO_CRS)
        # Should pick the geographic one which is irregular
        self.assertEqual('Geographic 2D CRS', gm.crs.type_name)
        self.assertEqual(False, gm.is_regular)

        gm = GridMapping.from_dataset(dataset, prefer_crs=GEO_CRS,
                                      prefer_is_regular=True)
        # Should pick the geographic one which is irregular
        self.assertEqual('Geographic 2D CRS', gm.crs.type_name)
        self.assertEqual(False, gm.is_regular)

    def test_no_grid_mapping_found(self):
        with self.assertRaises(ValueError) as cm:
            GridMapping.from_dataset(xr.Dataset())
        self.assertEqual('cannot find any grid mapping in dataset',
                         f'{cm.exception}')


# test/core/gridmapping/test_helpers.py

import unittest
from fractions import Fraction

from xcube.core.gridmapping.helpers import _to_int_or_float
from xcube.core.gridmapping.helpers import round_to_fraction


class RoundToFractionTest(unittest.TestCase):
    dump = False

    def test_invalid(self):
        with self.assertRaises(ValueError):
            round_to_fraction(0.29, digits=0)
        with self.assertRaises(ValueError):
            round_to_fraction(0.29, resolution=0)
        with self.assertRaises(ValueError):
            round_to_fraction(0.29, resolution=0.12)

    def test_1_025(self):
        def f(value):
            return float(round_to_fraction(value, 1, 0.25))

        self.assertAlmostEqual(-1, f(-1))
        self.assertAlmostEqual(0.0, f(0))
        self.assertAlmostEqual(1.0, f(1))
        self.assertAlmostEqual(1.25, f(1.2))
        self.assertAlmostEqual(1.25, f(1.3))
        self.assertAlmostEqual(1.5, f(1.4))
        self.assertAlmostEqual(1.5, f(1.45))
        self.assertAlmostEqual(1.5, f(1.51))
        self.assertAlmostEqual(1.75, f(1.7))
        self.assertAlmostEqual(2.0, f(1.9))
        self.assertAlmostEqual(2.0, f(1.96))
        self.assertAlmostEqual(2.0, f(1.98))
        self.assertAlmostEqual(2.0, f(2))

    def test_2_025(self):
        def f(value):
            return float(round_to_fraction(value, 2, 0.25))

        self.assertAlmostEqual(-1, f(-1))
        self.assertAlmostEqual(0.0, f(0))
        self.assertAlmostEqual(1.0, f(1))
        self.assertAlmostEqual(1.2, f(1.2))
        self.assertAlmostEqual(1.225, f(1.23))
        self.assertAlmostEqual(1.3, f(1.3))
        self.assertAlmostEqual(1.4, f(1.4))
        self.assertAlmostEqual(1.45, f(1.45))
        self.assertAlmostEqual(1.5, f(1.51))
        self.assertAlmostEqual(1.7, f(1.7))
        self.assertAlmostEqual(1.8, f(1.79))
        self.assertAlmostEqual(1.9, f(1.9))
        self.assertAlmostEqual(1.95, f(1.96))
        self.assertAlmostEqual(1.975, f(1.98))
        self.assertAlmostEqual(2.0, f(2))

    def test_default(self):
        values = [
            [-1.0, -1.0, Fraction(-1, 1)],
            [0.0, 0.0, Fraction(0, 1)],
            [5.247476065426347e-09, 5.2e-09, Fraction(13, 2500000000)],
            [3.427467229408875e-06, 3.4e-06, Fraction(17, 5000000)],
            [4.501758583626108e-06, 4.5e-06, Fraction(9, 2000000)],
            [1.1351705264714663e-05, 1.1e-05, Fraction(11, 1000000)],
            [0.00048171747406886744, 0.00048, Fraction(3, 6250)],
            [0.0018032657496927416, 0.0018, Fraction(9, 5000)],
            [0.0019897341919324425, 0.002, Fraction(1, 500)],
            [0.0041643509375105065, 0.0042, Fraction(21, 5000)],
            [0.030607346091352187, 0.031, Fraction(31, 1000)],
            [1.0076973439575128, 1.0, Fraction(1, 1)],
            [1.0, 1.0, Fraction(1, 1)],
            [84.54360269093455, 85.0, Fraction(85, 1)],
            [494.86581234602096, 490.0, Fraction(490, 1)],
            [987.9441243998718, 990.0, Fraction(990, 1)],
            [1757.368043916636, 1800.0, Fraction(1800, 1)],
            [1143506.2928512183, 1100000.0, Fraction(1100000, 1)],
            [217971970.75235566, 220000000.0, Fraction(220000000, 1)]
        ]
        self._assert_values(values, dict())

    def test_3_025(self):
        actual = round_to_fraction(1, digits=1, resolution=0.25)
        self.assertEqual(Fraction(1, 1), actual)
        values = [
            [-1.0, -1.0, Fraction(-1, 1)],
            [0.0, 0.0, Fraction(0, 1)],
            [5.247476065426347e-09, 5.2475e-09, Fraction(2099, 400000000000)],
            [3.427467229408875e-06, 3.4275e-06, Fraction(1371, 400000000)],
            [4.501758583626108e-06, 4.5025e-06, Fraction(1801, 400000000)],
            [1.1351705264714663e-05, 1.135e-05, Fraction(227, 20000000)],
            [0.00048171747406886744, 0.00048175, Fraction(1927, 4000000)],
            [0.0018032657496927416, 0.0018025, Fraction(721, 400000)],
            [0.0019897341919324425, 0.00199, Fraction(199, 100000)],
            [0.0041643509375105065, 0.004165, Fraction(833, 200000)],
            [0.030607346091352187, 0.0306, Fraction(153, 5000)],
            [1.0076973439575128, 1.0075, Fraction(403, 400)],
            [1.0, 1.0, Fraction(1, 1)],
            [84.54360269093455, 84.55, Fraction(1691, 20)],
            [494.86581234602096, 494.75, Fraction(1979, 4)],
            [987.9441243998718, 988.0, Fraction(988, 1)],
            [1757.368043916636, 1757.5, Fraction(3515, 2)],
            [1143506.2928512183, 1142500.0, Fraction(1142500, 1)],
            [217971970.75235566, 218000000.0, Fraction(218000000, 1)]
        ]
        self._assert_values(values, dict(digits=3, resolution=0.25))

    def test_2_5(self):
        values = [
            [-1.0, -1.0, Fraction(-1, 1)],
            [0.0, 0.0, Fraction(0, 1)],
            [5.247476065426347e-09, 5.25e-09, Fraction(21, 4000000000)],
            [3.427467229408875e-06, 3.45e-06, Fraction(69, 20000000)],
            [4.501758583626108e-06, 4.5e-06, Fraction(9, 2000000)],
            [1.1351705264714663e-05, 1.15e-05, Fraction(23, 2000000)],
            [0.00048171747406886744, 0.00048, Fraction(3, 6250)],
            [0.0018032657496927416, 0.0018, Fraction(9, 5000)],
            [0.0019897341919324425, 0.002, Fraction(1, 500)],
            [0.0041643509375105065, 0.00415, Fraction(83, 20000)],
            [0.030607346091352187, 0.0305, Fraction(61, 2000)],
            [1.0076973439575128, 1.0, Fraction(1, 1)],
            [1.0, 1.0, Fraction(1, 1)],
            [84.54360269093455, 84.5, Fraction(169, 2)],
            [494.86581234602096, 495.0, Fraction(495, 1)],
            [987.9441243998718, 990.0, Fraction(990, 1)],
            [1757.368043916636, 1750.0, Fraction(1750, 1)],
            [1143506.2928512183, 1150000.0, Fraction(1150000, 1)],
            [217971970.75235566, 220000000.0, Fraction(220000000, 1)]
        ]
        self._assert_values(values, dict(digits=2, resolution=0.5))

    def _assert_values(self, values, kwargs):
        if self.dump:
            results = []
            for value, _, _ in values:
                actual_fraction = round_to_fraction(value, **kwargs)
                results.append([value, float(actual_fraction), actual_fraction])
            import pprint
            pprint.pprint(results)

        for value, expected_float, expected_fraction in values:
            actual_fraction = round_to_fraction(value, **kwargs)
            self.assertEqual(expected_fraction, actual_fraction)
            self.assertAlmostEqual(expected_float, float(actual_fraction))


class ToIntOrFloatTest(unittest.TestCase):

    def test_down_to_int(self):
        result = _to_int_or_float(90.0001)
        self.assertEqual(90, result)

    def test_leave_as_bigger_float(self):
        result = _to_int_or_float(90.001)
        self.assertEqual(90.001, result)

    def test_up_to_int(self):
        result = _to_int_or_float(89.9999)
        self.assertEqual(90, result)

    def test_leave_as_smaller_float(self):
        result = _to_int_or_float(89.999)
        self.assertEqual(89.999, result)

    def test_up_to_int_small_value(self):
        result = _to_int_or_float(0.99999)
        self.assertEqual(1, result)

    def test_leave_as_smaller_float_small_value(self):
        result = _to_int_or_float(0.9999)
        self.assertEqual(0.9999, result)


# test/core/gridmapping/test_regular.py

import unittest

import numpy as np
import pyproj
import xarray as xr

from xcube.core.gridmapping import CRS_WGS84
from xcube.core.gridmapping import GridMapping
from xcube.core.gridmapping.regular import RegularGridMapping

# noinspection PyProtectedMember

GEO_CRS = pyproj.crs.CRS(4326)
NOT_A_GEO_CRS = pyproj.crs.CRS(5243)


# noinspection PyMethodMayBeStatic
class RegularGridMappingTest(unittest.TestCase):

    def test_default_props(self):
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, CRS_WGS84)
        self.assertEqual((1000, 1000), gm.size)
        self.assertEqual((1000, 1000), gm.tile_size)
        self.assertEqual(10, gm.x_min)
        self.assertEqual(53, gm.y_min)
        self.assertEqual((0.01, 0.01), gm.xy_res)
        self.assertEqual(True, gm.is_regular)
        self.assertEqual(False, gm.is_j_axis_up)

    def test_invalid_y(self):
        with self.assertRaises(ValueError) as cm:
            GridMapping.regular((1000, 1000), (10, -90.5), 0.01, CRS_WGS84)
        self.assertEqual('invalid y_min', f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            GridMapping.regular((1000, 1000), (10, 53), 0.1, CRS_WGS84)
        self.assertEqual('invalid size, y_min combination', f'{cm.exception}')

    def test_xy_bbox(self):
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, CRS_WGS84)
        self.assertEqual((10, 53, 20, 63), gm.xy_bbox)
        self.assertEqual(False, gm.is_lon_360)

    def test_xy_bbox_anti_meridian(self):
        gm = GridMapping.regular((2000, 1000), (174.0, -30.0), 0.005, CRS_WGS84)
        self.assertEqual((174.0, -30.0, 184.0, -25.0), gm.xy_bbox)
        self.assertEqual(True, gm.is_lon_360)

    def test_derive(self):
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, CRS_WGS84)
        self.assertEqual((1000, 1000), gm.size)
        self.assertEqual((1000, 1000), gm.tile_size)
        self.assertEqual(False, gm.is_j_axis_up)
        derived_gm = gm.derive(tile_size=500, is_j_axis_up=True)
        self.assertIsNot(gm, derived_gm)
        self.assertIsInstance(derived_gm, RegularGridMapping)
        self.assertEqual((1000, 1000), derived_gm.size)
        self.assertEqual((500, 500), derived_gm.tile_size)
        self.assertEqual(True, derived_gm.is_j_axis_up)

    def test_xy_coords(self):
        gm = GridMapping.regular((8, 4), (10, 53), 0.1, CRS_WGS84).derive(tile_size=(4, 2))
        xy_coords = gm.xy_coords
        self.assertIsInstance(xy_coords, xr.DataArray)
        self.assertIs(gm.xy_coords, xy_coords)
        self.assertEqual(('coord', 'lat', 'lon'), xy_coords.dims)
        self.assertEqual((2, 4, 8), xy_coords.shape)
        self.assertEqual(((2,), (2, 2), (4, 4)), xy_coords.chunks)
        np.testing.assert_almost_equal(
            np.array([
                [10.05, 10.15, 10.25, 10.35, 10.45, 10.55, 10.65, 10.75],
                [10.05, 10.15, 10.25, 10.35, 10.45, 10.55, 10.65, 10.75],
                [10.05, 10.15, 10.25, 10.35, 10.45, 10.55, 10.65, 10.75],
                [10.05, 10.15, 10.25, 10.35, 10.45, 10.55, 10.65, 10.75]
            ]),
            xy_coords.values[0]
        )
        np.testing.assert_almost_equal(
            np.array([
                [53.35, 53.35, 53.35, 53.35, 53.35, 53.35, 53.35, 53.35],
                [53.25, 53.25, 53.25, 53.25, 53.25, 53.25, 53.25, 53.25],
                [53.15, 53.15, 53.15, 53.15, 53.15, 53.15, 53.15, 53.15],
                [53.05, 53.05, 53.05, 53.05, 53.05, 53.05, 53.05, 53.05]
            ]),
            xy_coords.values[1]
        )

    def test_xy_names(self):
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, GEO_CRS).derive(tile_size=500)
        self.assertEqual(('lon', 'lat'), gm.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm.xy_dim_names)
        gm = GridMapping.regular((1000, 1000), (10, 53), 0.01, NOT_A_GEO_CRS).derive(tile_size=500)
        self.assertEqual(('x', 'y'), gm.xy_var_names)
        self.assertEqual(('x', 'y'), gm.xy_dim_names)

    def test_ij_bboxes(self):
        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS)
        np.testing.assert_almost_equal(gm.ij_bboxes,
                                       np.array([[0, 0, 2000, 1000]],
                                                dtype=np.int64))

        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS).derive(tile_size=500)
        np.testing.assert_almost_equal(gm.ij_bboxes,
                                       np.array([
                                           [0, 0, 500, 500],
                                           [500, 0, 1000, 500],
                                           [1000, 0, 1500, 500],
                                           [1500, 0, 2000, 500],
                                           [0, 500, 500, 1000],
                                           [500, 500, 1000, 1000],
                                           [1000, 500, 1500, 1000],
                                           [1500, 500, 2000, 1000]
                                       ], dtype=np.int64))

    def test_xy_bboxes(self):
        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS)
        np.testing.assert_almost_equal(gm.xy_bboxes,
                                       np.array([[10., 20., 210., 120.]],
                                                dtype=np.float64))

        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS).derive(tile_size=500)
        np.testing.assert_almost_equal(gm.xy_bboxes,
                                       np.array([
                                           [10., 70, 60, 120.],
                                           [60., 70, 110, 120.],
                                           [110., 70, 160, 120.],
                                           [160., 70, 210, 120.],
                                           [10., 20, 60, 70.],
                                           [60., 20, 110, 70.],
                                           [110., 20, 160, 70.],
                                           [160., 20, 210, 70.]
                                       ], dtype=np.float64))

    def test_xy_bboxes_is_j_axis_up(self):
        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS).derive(is_j_axis_up=True)
        np.testing.assert_almost_equal(gm.xy_bboxes,
                                       np.array([[10., 20., 210., 120.]],
                                                dtype=np.float64))

        gm = GridMapping.regular(size=(2000, 1000),
                                 xy_min=(10.0, 20.0),
                                 xy_res=0.1,
                                 crs=NOT_A_GEO_CRS, ).derive(tile_size=500,
                                                             is_j_axis_up=True)
        np.testing.assert_almost_equal(gm.xy_bboxes,
                                       np.array([
                                           [10., 20., 60., 70.],
                                           [60., 20., 110., 70.],
                                           [110., 20., 160., 70.],
                                           [160., 20., 210., 70.],
                                           [10., 70., 60., 120.],
                                           [60., 70., 110., 120.],
                                           [110., 70., 160., 120.],
                                           [160., 70., 210., 120.]
                                       ], dtype=np.float64))

    def test_to_coords(self):
        gm = GridMapping.regular(size=(10, 6),
                                 xy_min=(-2600.0, 1200.0),
                                 xy_res=10.0,
                                 crs=NOT_A_GEO_CRS)

        cv = gm.to_coords(xy_var_names=('x', 'y'))
        self._assert_coord_vars(cv,
                                (10, 6),
                                ('x', 'y'),
                                (-2595., -2505.),
                                (1255., 1205.),
                                ('x_bnds', 'y_bnds'),
                                (
                                    (-2600., -2590.),
                                    (-2510., -2500.),
                                ),
                                (
                                    (1260., 1250.),
                                    (1210., 1200.),
                                ))

    def test_coord_vars_j_axis_up(self):
        gm = GridMapping.regular(size=(10, 6),
                                 xy_min=(-2600.0, 1200.0),
                                 xy_res=10.0,
                                 crs=NOT_A_GEO_CRS).derive(is_j_axis_up=True)

        cv = gm.to_coords(xy_var_names=('x', 'y'))
        self._assert_coord_vars(cv,
                                (10, 6),
                                ('x', 'y'),
                                (-2595., -2505.),
                                (1205., 1255.),
                                ('x_bnds', 'y_bnds'),
                                (
                                    (-2600., -2590.),
                                    (-2510., -2500.),
                                ),
                                (
                                    (1200., 1210.),
                                    (1250., 1260.),
                                ))

    def test_coord_vars_antimeridian(self):
        gm = GridMapping.regular(size=(10, 10),
                                 xy_min=(172.0, 53.0),
                                 xy_res=2.0,
                                 crs=GEO_CRS)

        cv = gm.to_coords(xy_var_names=('lon', 'lat'))
        self._assert_coord_vars(cv,
                                (10, 10),
                                ('lon', 'lat'),
                                (173.0, -169.0),
                                (72.0, 54.0),
                                ('lon_bnds', 'lat_bnds'),
                                (
                                    (172., 174.),
                                    (-170., -168.),
                                ),
                                (
                                    (73., 71.),
                                    (55., 53.),
                                ))

    def _assert_coord_vars(self,
                           cv,
                           size,
                           xy_names,
                           x_values,
                           y_values,
                           xy_bnds_names,
                           x_bnds_values,
                           y_bnds_values):
        self.assertIsNotNone(cv)
        self.assertIn(xy_names[0], cv)
        self.assertIn(xy_names[1], cv)
        self.assertIn(xy_bnds_names[0], cv)
        self.assertIn(xy_bnds_names[1], cv)

        x = cv[xy_names[0]]
        self.assertEqual((size[0],), x.shape)
        np.testing.assert_almost_equal(x.values[0], np.array(x_values[0]))
        np.testing.assert_almost_equal(x.values[-1], np.array(x_values[-1]))

        y = cv[xy_names[1]]
        self.assertEqual((size[1],), y.shape)
        np.testing.assert_almost_equal(y.values[0], np.array(y_values[0]))
        np.testing.assert_almost_equal(y.values[-1], np.array(y_values[-1]))

        x_bnds = cv[xy_bnds_names[0]]
        self.assertEqual((size[0], 2), x_bnds.shape)
        np.testing.assert_almost_equal(x_bnds.values[0], np.array(x_bnds_values[0]))
        np.testing.assert_almost_equal(x_bnds.values[-1], np.array(x_bnds_values[-1]))

        y_bnds = cv[xy_bnds_names[1]]
        self.assertEqual((size[1], 2), y_bnds.shape)
        np.testing.assert_almost_equal(y_bnds.values[0], y_bnds_values[0])
        np.testing.assert_almost_equal(y_bnds.values[-1], y_bnds_values[-1])


# test/core/gridmapping/test_transform.py

import unittest

import pyproj.crs

from test.sampledata import create_s2plus_dataset
from xcube.core.gridmapping import CRS_CRS84
from xcube.core.gridmapping import CRS_WGS84
from xcube.core.gridmapping import GridMapping

CRS_UTM_32N = pyproj.crs.CRS(32632)

import numpy as np


# noinspection PyMethodMayBeStatic
class TransformTest(unittest.TestCase):

    def test_transform(self):
        gm = GridMapping.regular(size=(3, 3), xy_min=(10, 53), xy_res=0.1, crs=CRS_CRS84)
        gm_t = gm.transform(crs=CRS_UTM_32N)

        self.assertEqual(CRS_UTM_32N, gm_t.crs)
        self.assertEqual(False, gm_t.is_regular)
        self.assertEqual(('transformed_x', 'transformed_y'), gm_t.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm_t.xy_dim_names)
        np.testing.assert_almost_equal(
            np.array([
                [570057.076286, 576728.9360228, 583400.7295284],
                [570220.3304187, 576907.7404859, 583595.0849538],
                [570383.3684844, 577086.3083212, 583789.1831954]
            ]),
            gm_t.xy_coords[0])
        np.testing.assert_almost_equal(
            np.array([
                [5900595.928991, 5900698.5746648, 5900810.5532744],
                [5889471.9033896, 5889574.6540572, 5889686.7472201],
                [5878348.0594403, 5878450.9138481, 5878563.1201969]
            ]),
            gm_t.xy_coords[1])

    def test_transform_xy_var_names(self):
        gm = GridMapping.regular(size=(3, 3), xy_min=(10, 53), xy_res=0.1, crs=CRS_CRS84)
        gm_t = gm.transform(crs=CRS_UTM_32N, xy_var_names=('x', 'y'))
        self.assertEqual(CRS_UTM_32N, gm_t.crs)
        self.assertEqual(('x', 'y'), gm_t.xy_var_names)
        self.assertEqual(('lon', 'lat'), gm_t.xy_dim_names)

    def test_transform_s2(self):
        dataset = create_s2plus_dataset()

        gm = GridMapping.from_dataset(dataset, prefer_is_regular=True)
        # Assert we've picked the projected one which is regular
        self.assertEqual("Derived Projected CRS", gm.crs.type_name)
        self.assertEqual(True, gm.is_regular)

        gm_t = gm.transform(CRS_CRS84)
        self.assertEqual(CRS_CRS84, gm_t.crs)

        gm_t = gm.transform(CRS_WGS84)
        self.assertEqual(CRS_WGS84, gm_t.crs)

    def test_transform_no_op(self):
        dataset = create_s2plus_dataset()

        gm = GridMapping.from_dataset(dataset, prefer_is_regular=True)
        # Assert we've picked the projected one which is regular
        self.assertEqual("Derived Projected CRS", gm.crs.type_name)
        self.assertEqual(True, gm.is_regular)

        gm_t = gm.transform(gm.crs)
        self.assertIs(gm, gm_t)

        # Almost no op
        gm = GridMapping.regular(size=(3, 3), xy_min=(10, 53), xy_res=0.1, crs=CRS_CRS84)
        gm_t = gm.transform(crs=gm.crs, xy_var_names=('x', 'y'))
        self.assertEqual(('x', 'y'), gm_t.xy_var_names)



# test/core/resampling/test_affine.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.gridmapping import GridMapping
from xcube.core.resampling import affine_transform_dataset

nan = np.nan

res = 0.1

source_ds = xr.Dataset(
    data_vars=dict(
        refl=xr.DataArray(
            np.array([
                [0, 1, 0, 2, 0, 3, 0, 4],
                [2, 0, 3, 0, 4, 0, 1, 0],
                [0, 4, 0, 1, 0, 2, 0, 3],
                [1, 0, 2, 0, 3, 0, 4, 0],
                [0, 3, 0, 4, 0, 1, 0, 2],
                [4, 0, 1, 0, 2, 0, 3, 0],
            ], dtype=np.float64),
            dims=('lat', 'lon'))
    ),
    coords=dict(
        lon=xr.DataArray(50.0 + res * np.arange(0, 8) + 0.5 * res, dims='lon'),
        lat=xr.DataArray(10.6 - res * np.arange(0, 6) - 0.5 * res, dims='lat')
    )
)

source_gm = GridMapping.from_dataset(source_ds)


class AffineTransformDatasetTest(unittest.TestCase):
    def test_subset(self):
        target_gm = GridMapping.regular((3, 3), (50.0, 10.0), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((3, 3), target_ds.refl.shape)
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [1, 0, 2],
                [0, 3, 0],
                [4, 0, 1],
            ]))

        target_gm = GridMapping.regular((3, 3), (50.1, 10.1), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((3, 3), target_ds.refl.shape)
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [4, 0, 1],
                [0, 2, 0],
                [3, 0, 4],
            ]))

        target_gm = GridMapping.regular((3, 3), (50.05, 10.05), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((3, 3), target_ds.refl.shape)
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [1.25, 1.5, 0.75],
                [1., 1.25, 1.5],
                [1.75, 1., 1.25]
            ]))

    def test_downscale_x2(self):
        target_gm = GridMapping.regular((8, 6), (50, 10), 2 * res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [0.75, 1.25, 1.75, 1.25, nan, nan, nan, nan],
                [1.25, 0.75, 1.25, 1.75, nan, nan, nan, nan],
                [1.75, 1.25, 0.75, 1.25, nan, nan, nan, nan]
            ]))

    def test_downscale_x2_and_shift(self):
        target_gm = GridMapping.regular((8, 6), (49.8, 9.8), 2 * res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, 0.75, 1.25, 1.75, 1.25, nan, nan, nan],
                [nan, 1.25, 0.75, 1.25, 1.75, nan, nan, nan],
                [nan, 1.75, 1.25, 0.75, 1.25, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan]
            ]))

    def test_upscale_x2(self):
        target_gm = GridMapping.regular((8, 6), (50, 10), res / 2, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [1.0, 0.5, 0.0, 1.0, 2.0, 1.0, 0.0, 1.5],
                [0.5, 1.0, 1.5, 1.25, 1.0, 1.5, 2.0, 1.75],
                [0.0, 1.5, 3.0, 1.5, 0.0, 2.0, 4.0, 2.0],
                [2.0, 1.75, 1.5, 1.0, 0.5, 1.25, 2.0, 1.5],
                [4.0, 2.0, 0.0, 0.5, 1.0, 0.5, 0.0, 1.0],
                [nan, nan, nan, nan, nan, nan, nan, nan]
            ]))

    def test_upscale_x2_and_shift(self):
        target_gm = GridMapping.regular((8, 6), (49.9, 9.95), res / 2, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, 0.5, 1.0, 1.5, 1.25, 1.0, 1.5],
                [nan, nan, 0.0, 1.5, 3.0, 1.5, 0.0, 2.0],
                [nan, nan, 2.0, 1.75, 1.5, 1.0, 0.5, 1.25],
                [nan, nan, 4.0, 2.0, 0.0, 0.5, 1.0, 0.5],
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [nan, nan, nan, nan, nan, nan, nan, nan]
            ]))

    def test_shift(self):
        target_gm = GridMapping.regular((8, 6), (50.2, 10.1), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, nan, nan, nan, nan, nan, nan],
                [0.0, 2.0, 0.0, 3.0, 0.0, 4.0, nan, nan],
                [3.0, 0.0, 4.0, 0.0, 1.0, 0.0, nan, nan],
                [0.0, 1.0, 0.0, 2.0, 0.0, 3.0, nan, nan],
                [2.0, 0.0, 3.0, 0.0, 4.0, 0.0, nan, nan],
                [0.0, 4.0, 0.0, 1.0, 0.0, 2.0, nan, nan]
            ]))

        target_gm = GridMapping.regular((8, 6), (49.8, 9.9), res, source_gm.crs)
        target_ds = affine_transform_dataset(source_ds, source_gm, target_gm)
        self.assertIsInstance(target_ds, xr.Dataset)
        self.assertEqual(set(target_ds.variables), set(source_ds.variables))
        self.assertEqual((6, 8), target_ds.refl.shape)
        print(repr(target_ds.refl.values))
        np.testing.assert_almost_equal(
            target_ds.refl.values,
            np.array([
                [nan, nan, 2.0, 0.0, 3.0, 0.0, 4.0, 0.0],
                [nan, nan, 0.0, 4.0, 0.0, 1.0, 0.0, 2.0],
                [nan, nan, 1.0, 0.0, 2.0, 0.0, 3.0, 0.0],
                [nan, nan, 0.0, 3.0, 0.0, 4.0, 0.0, 1.0],
                [nan, nan, 4.0, 0.0, 1.0, 0.0, 2.0, 0.0],
                [nan, nan, nan, nan, nan, nan, nan, nan]
            ]))


# test/core/resampling/test_rectify.py

import unittest
from typing import Tuple

import numpy as np
# noinspection PyUnresolvedReferences
import xarray as xr

from test.sampledata import SourceDatasetMixin
from test.sampledata import create_s2plus_dataset
from xcube.core.gridmapping import CRS_WGS84
from xcube.core.gridmapping import GridMapping
from xcube.core.resampling import rectify_dataset

nan = np.nan


# noinspection PyMethodMayBeStatic
class RectifyDatasetTest(SourceDatasetMixin, unittest.TestCase):

    def test_rectify_2x2_to_default(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(4, 4),
                                        xy_min=(-1, 49),
                                        xy_res=2,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        # target_ds = rectify_dataset(source_ds)

        rad = target_ds.rad
        # lon, lat, rad = self._assert_shape_and_dim(target_ds, (4, 4))
        # np.testing.assert_almost_equal(lon.values,
        #                                np.array([0., 2., 4., 6.],
        #                                         dtype=lon.dtype))
        # np.testing.assert_almost_equal(lat.values,
        #                                np.array([56., 54., 52., 50.],
        #                                         dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values,
                                       np.array([
                                           [nan, nan, nan, nan],
                                           [nan, 1.0, 2.0, nan],
                                           [3.0, 3.0, 2.0, nan],
                                           [nan, 4.0, nan, nan]
                                       ],
                                           dtype=rad.dtype))

    def test_rectify_2x2_to_7x7(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(7, 7),
                                        xy_min=(-0.5, 49.5),
                                        xy_res=1.0,
                                        crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (7, 7))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5., 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55., 54., 53., 52., 51., 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values,
                                       np.array([
                                           [nan, 1.0, nan, nan, nan, nan, nan],
                                           [nan, 1.0, 1.0, nan, nan, nan, nan],
                                           [nan, 1.0, 1.0, 1.0, 2.0, nan, nan],
                                           [nan, 3.0, 3.0, 1.0, 2.0, 2.0, 2.0],
                                           [3.0, 3.0, 3.0, 4.0, 2.0, nan, nan],
                                           [nan, 3.0, 4.0, 4.0, nan, nan, nan],
                                           [nan, nan, 4.0, nan, nan, nan, nan],
                                       ], dtype=rad.dtype))

    def test_rectify_2x2_to_7x7_subset(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(7, 7),
                                        xy_min=(1.5, 50.5),
                                        xy_res=1.0,
                                        crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        lon, lat, rad = self._assert_shape_and_dim(target_ds, (7, 7))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([2.0, 3.0, 4.0, 5., 6., 7., 8.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([57., 56., 55., 54., 53., 52., 51.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values,
                                       np.array([
                                           [nan, nan, nan, nan, nan, nan, nan],
                                           [nan, nan, nan, nan, nan, nan, nan],
                                           [1.0, nan, nan, nan, nan, nan, nan],
                                           [1.0, 1.0, 2.0, nan, nan, nan, nan],
                                           [3.0, 1.0, 2.0, 2.0, 2.0, nan, nan],
                                           [3.0, 4.0, 2.0, nan, nan, nan, nan],
                                           [4.0, 4.0, nan, nan, nan, nan, nan],
                                       ], dtype=rad.dtype))

    def test_rectify_2x2_to_13x13(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_j_axis_up(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        is_j_axis_up=True)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([50., 50.5, 51., 51.5, 52., 52.5, 53., 53.5, 54., 54.5, 55.,
                                                 55.5, 56.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype)[::-1])

    def test_rectify_2x2_to_13x13_j_axis_up_dask_5x5(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=5,
                                        is_j_axis_up=True)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((5, 5, 3), (5, 5, 3)))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([50., 50.5, 51., 51.5, 52., 52.5, 53., 53.5, 54., 54.5, 55.,
                                                 55.5, 56.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype)[::-1])

    def test_rectify_2x2_to_13x13_dask_7x7(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=7)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((7, 6), (7, 6)))

        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_dask_5x5(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=5)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((5, 5, 3), (5, 5, 3)))

        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_dask_3x13(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=(3, 13))

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((13,), (3, 3, 3, 3, 1)))

        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_dask_13x3(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=(13, 3))

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13), chunks=((3, 3, 3, 3, 1), (13,)))

        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_antimeridian(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords_antimeridian()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(177.75, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)

        self.assertEqual(True, target_gm.is_lon_360)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm)

        self.assertIsNotNone(target_ds)
        lon, lat, rad = self._assert_shape_and_dim(target_ds, (13, 13))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([178., 178.5, 179., 179.5, 180., -179.5, -179., -178.5,
                                                 -178., -177.5, -177., -176.5, -176.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_output_ij_names(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm, output_ij_names=('source_i', 'source_j'))

        lon, lat, rad, source_i, source_j = self._assert_shape_and_dim(target_ds, (13, 13),
                                                                       var_names=('rad', 'source_i', 'source_j'))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))
        np.testing.assert_almost_equal(np.floor(source_i.values + 0.5), self.expected_i_13x13())
        np.testing.assert_almost_equal(np.floor(source_j.values + 0.5), self.expected_j_13x13())

    def test_rectify_2x2_to_13x13_output_ij_names_dask(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-0.25, 49.75),
                                        xy_res=0.5,
                                        crs=CRS_WGS84,
                                        tile_size=5)

        target_ds = rectify_dataset(source_ds, target_gm=target_gm, output_ij_names=('source_i', 'source_j'))
        lon, lat, rad, source_i, source_j = self._assert_shape_and_dim(target_ds, (13, 13),
                                                                       chunks=((5, 5, 3), (5, 5, 3)),
                                                                       var_names=('rad', 'source_i', 'source_j'))
        np.testing.assert_almost_equal(lon.values,
                                       np.array([0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5., 5.5, 6.],
                                                dtype=lon.dtype))
        np.testing.assert_almost_equal(lat.values,
                                       np.array([56., 55.5, 55., 54.5, 54., 53.5, 53., 52.5, 52., 51.5, 51.,
                                                 50.5, 50.],
                                                dtype=lat.dtype))
        # print('source_i:', np.floor(source_i.values + 0.5))
        # print('source_j:', np.floor(source_j.values + 0.5))
        np.testing.assert_almost_equal(np.floor(source_i.values + 0.5), self.expected_i_13x13())
        np.testing.assert_almost_equal(np.floor(source_j.values + 0.5), self.expected_j_13x13())
        np.testing.assert_almost_equal(rad.values, self.expected_rad_13x13(rad.dtype))

    def test_rectify_2x2_to_13x13_none(self):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(10.0, 50.0),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        self.assertIsNone(target_ds)

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(-10.0, 50.0),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        self.assertIsNone(target_ds)

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(0.0, 58.0),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        self.assertIsNone(target_ds)

        target_gm = GridMapping.regular(size=(13, 13),
                                        xy_min=(0.0, 42.0),
                                        xy_res=0.5,
                                        crs=CRS_WGS84)
        target_ds = rectify_dataset(source_ds, target_gm=target_gm)
        self.assertIsNone(target_ds)

    def _assert_shape_and_dim(self, target_ds, size, chunks=None, var_names=('rad',)) -> Tuple[xr.DataArray, ...]:
        w, h = size

        self.assertIn('lon', target_ds)
        lon = target_ds['lon']
        self.assertEqual((w,), lon.shape)
        self.assertEqual(('lon',), lon.dims)

        self.assertIn('lat', target_ds)
        lat = target_ds['lat']
        self.assertEqual((h,), lat.shape)
        self.assertEqual(('lat',), lat.dims)

        # noinspection PyShadowingBuiltins
        vars = []
        for var_name in var_names:
            self.assertIn(var_name, target_ds)
            var = target_ds[var_name]
            self.assertEqual((h, w), var.shape)
            self.assertEqual(('lat', 'lon'), var.dims)
            self.assertEqual(chunks, var.chunks)
            vars.append(var)

        return (lon, lat, *vars)

    def test_compute_and_extract_source_pixels(self):
        from xcube.core.resampling.rectify import _compute_ij_images_numpy_parallel
        from xcube.core.resampling.rectify import _compute_var_image_numpy_parallel
        self._assert_compute_and_extract_source_pixels(_compute_ij_images_numpy_parallel,
                                                       _compute_var_image_numpy_parallel, False)
        from xcube.core.resampling.rectify import _compute_ij_images_numpy_sequential
        from xcube.core.resampling.rectify import _compute_var_image_numpy_sequential
        self._assert_compute_and_extract_source_pixels(_compute_ij_images_numpy_sequential,
                                                       _compute_var_image_numpy_sequential, False)

    def test_compute_and_extract_source_pixels_j_axis_up(self):
        from xcube.core.resampling.rectify import _compute_ij_images_numpy_parallel
        from xcube.core.resampling.rectify import _compute_var_image_numpy_parallel
        self._assert_compute_and_extract_source_pixels(_compute_ij_images_numpy_parallel,
                                                       _compute_var_image_numpy_parallel, True)
        from xcube.core.resampling.rectify import _compute_ij_images_numpy_sequential
        from xcube.core.resampling.rectify import _compute_var_image_numpy_sequential
        self._assert_compute_and_extract_source_pixels(_compute_ij_images_numpy_sequential,
                                                       _compute_var_image_numpy_sequential, True)

    def _assert_compute_and_extract_source_pixels(self,
                                                  compute_ij_images,
                                                  compute_var_image,
                                                  is_j_axis_up: bool):
        source_ds = self.new_2x2_dataset_with_irregular_coords()

        dst_src_ij = np.full((2, 13, 13), np.nan, dtype=np.float64)
        dst_x_offset = -0.25
        dst_y_offset = 49.75 if is_j_axis_up else 56.25
        dst_x_scale = 0.5
        dst_y_scale = 0.5 if is_j_axis_up else -0.5
        compute_ij_images(source_ds.lon.values,
                          source_ds.lat.values,
                          0,
                          0,
                          dst_src_ij,
                          dst_x_offset,
                          dst_y_offset,
                          dst_x_scale,
                          dst_y_scale,
                          1e-5)

        if not is_j_axis_up:
            np.testing.assert_almost_equal(np.floor(dst_src_ij[0] + 0.5), self.expected_i_13x13())
            np.testing.assert_almost_equal(np.floor(dst_src_ij[1] + 0.5), self.expected_j_13x13())
        else:
            np.testing.assert_almost_equal(np.floor(dst_src_ij[0] + 0.5), self.expected_i_13x13()[::-1])
            np.testing.assert_almost_equal(np.floor(dst_src_ij[1] + 0.5), self.expected_j_13x13()[::-1])

        target_rad = np.full((13, 13), np.nan, dtype=np.float64)

        compute_var_image(source_ds.rad.values,
                          dst_src_ij,
                          target_rad)

        if not is_j_axis_up:
            np.testing.assert_almost_equal(target_rad, self.expected_rad_13x13(target_rad.dtype))
        else:
            np.testing.assert_almost_equal(target_rad, self.expected_rad_13x13(target_rad.dtype)[::-1])

    def expected_i_13x13(self):
        return np.array([
            [nan, nan, 0.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, 0.0, 0.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, nan, nan, nan, nan, nan],
            [nan, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, nan, nan, nan, nan],
            [nan, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, nan, nan],
            [nan, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
            [nan, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan],
            [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan],
            [nan, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, nan, 1.0, nan, nan, nan, nan, nan, nan, nan, nan],
        ],
            dtype=np.float64)

    def expected_j_13x13(self):
        return np.array([
            [nan, nan, 0.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, 0.0, 0.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, nan, nan, nan, nan, nan],
            [nan, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, nan, nan, nan, nan],
            [nan, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, nan, nan],
            [nan, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
            [nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, nan, nan],
            [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, nan, nan, nan],
            [nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, nan, 1.0, nan, nan, nan, nan, nan, nan, nan, nan],
        ],
            dtype=np.float64)

    def expected_rad_13x13(self, dtype):
        return np.array([
            [nan, nan, 1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, nan, nan, nan, nan, nan],
            [nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, nan, nan, nan, nan],
            [nan, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, nan, nan],
            [nan, 3.0, 3.0, 3.0, 3.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0],
            [nan, 3.0, 3.0, 3.0, 3.0, 3.0, 1.0, 2.0, 2.0, 2.0, 2.0, nan, nan],
            [3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 2.0, 2.0, 2.0, nan, nan, nan],
            [nan, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 2.0, nan, nan, nan, nan],
            [nan, nan, 3.0, 4.0, 4.0, 4.0, 4.0, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, 4.0, 4.0, 4.0, nan, nan, nan, nan, nan, nan, nan],
            [nan, nan, nan, nan, 4.0, nan, nan, nan, nan, nan, nan, nan, nan],
        ], dtype=dtype)


class RectifySentinel2DatasetTest(SourceDatasetMixin, unittest.TestCase):

    def test_rectify_dataset(self):
        source_ds = create_s2plus_dataset()

        expected_data = np.array([
            [nan, nan, nan, nan, nan, nan, nan, nan, nan],
            [nan, 0.019001, 0.019001, 0.008999, 0.012001, 0.012001, 0.022999, nan, nan],
            [nan, 0.021, 0.021, 0.009998, 0.009998, 0.008999, 0.022999, nan, nan],
            [nan, 0.022999, 0.022999, 0.007999, 0.007999, 0.008999, 0.023998, nan, nan],
            [nan, 0.022999, 0.022999, 0.007, 0.007, 0.009998, 0.021, nan, nan],
            [nan, nan, nan, nan, nan, nan, nan, nan, nan]
        ])

        source_gm = GridMapping.from_dataset(source_ds, prefer_crs=CRS_WGS84)

        target_ds = rectify_dataset(source_ds,
                                    source_gm=source_gm,
                                    tile_size=None)
        self.assertEqual(None, target_ds.rrs_665.chunks)
        print(repr(target_ds.rrs_665.values))
        np.testing.assert_almost_equal(target_ds.rrs_665.values,
                                       expected_data, decimal=3)

        target_ds = rectify_dataset(source_ds,
                                    source_gm=source_gm,
                                    tile_size=5)
        self.assertEqual(((5, 1), (5, 4)), target_ds.rrs_665.chunks)
        np.testing.assert_almost_equal(target_ds.rrs_665.values,
                                       expected_data, decimal=3)

        target_ds = rectify_dataset(source_ds,
                                    source_gm=source_gm,
                                    tile_size=None,
                                    is_j_axis_up=True)
        self.assertEqual(None, target_ds.rrs_665.chunks)
        np.testing.assert_almost_equal(target_ds.rrs_665.values,
                                       expected_data[::-1], decimal=3)

        target_ds = rectify_dataset(source_ds,
                                    source_gm=source_gm,
                                    tile_size=5,
                                    is_j_axis_up=True)
        self.assertEqual(((5, 1), (5, 4)), target_ds.rrs_665.chunks)
        np.testing.assert_almost_equal(target_ds.rrs_665.values,
                                       expected_data[::-1], decimal=3)


# test/core/resampling/test_spatial.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.gridmapping import CRS_WGS84
from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.resampling import resample_in_space

nan = np.nan


class ResampleInSpaceTest(unittest.TestCase):
    # TODO: write more tests

    # noinspection PyMethodMayBeStatic
    def test_affine_transform_dataset(self):
        source_ds = new_cube()
        source_gm = GridMapping.from_dataset(source_ds)
        target_gm = GridMapping.regular(size=(8, 4),
                                        xy_min=(0, 0),
                                        xy_res=2,
                                        crs=CRS_WGS84)

        # TODO: assert stuff
        resample_in_space(source_ds, source_gm, target_gm)


# test/core/resampling/test_temporal.py

import unittest

import numpy as np
import pandas as pd

from test.sampledata import new_test_dataset
from xcube.core.chunk import chunk_dataset
from xcube.core.resampling import resample_in_time
from xcube.core.schema import CubeSchema


class ResampleInTimeTest(unittest.TestCase):

    def setUp(self) -> None:
        num_times = 30

        time = []
        periods = ['1D', '1D', '3D', '4D', '2D']
        t = pd.to_datetime('2017-07-01T10:30:15Z', utc=True)
        for i in range(num_times):
            time.append(t.isoformat())
            t += pd.to_timedelta(periods[i % len(periods)])

        temperature, precipitation = zip(*[(272 + 0.1 * i, 120 - 0.2 * i) for i in range(num_times)])

        input_cube = new_test_dataset(time, temperature=temperature, precipitation=precipitation)
        input_cube = chunk_dataset(input_cube, chunk_sizes=dict(time=1, lat=90, lon=180))
        self.input_cube = input_cube

    def test_resample_in_time_min_max(self):
        resampled_cube = resample_in_time(self.input_cube, '2W', ['min', 'max'])
        self.assertIsNot(resampled_cube, self.input_cube)
        self.assertIn('time', resampled_cube)
        self.assertIn('temperature_min', resampled_cube)
        self.assertIn('temperature_max', resampled_cube)
        self.assertIn('precipitation_min', resampled_cube)
        self.assertIn('precipitation_max', resampled_cube)
        self.assertEqual(('time',), resampled_cube.time.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_min.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_max.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_min.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_max.dims)
        self.assertEqual((6,), resampled_cube.time.shape)
        self.assertEqual((6, 180, 360), resampled_cube.temperature_min.shape)
        self.assertEqual((6, 180, 360), resampled_cube.temperature_max.shape)
        self.assertEqual((6, 180, 360), resampled_cube.precipitation_min.shape)
        self.assertEqual((6, 180, 360), resampled_cube.precipitation_max.shape)
        np.testing.assert_equal(resampled_cube.time.values,
                                np.array(
                                    ['2017-06-25T00:00:00', '2017-07-09T00:00:00',
                                     '2017-07-23T00:00:00', '2017-08-06T00:00:00',
                                     '2017-08-20T00:00:00', '2017-09-03T00:00:00'], dtype=np.datetime64))
        np.testing.assert_allclose(resampled_cube.temperature_min.values[..., 0, 0],
                                   np.array([272.0, 272.4, 273.0, 273.8, 274.4, 274.9]))
        np.testing.assert_allclose(resampled_cube.temperature_max.values[..., 0, 0],
                                   np.array([272.3, 272.9, 273.7, 274.3, 274.8, 274.9]))
        np.testing.assert_allclose(resampled_cube.precipitation_min.values[..., 0, 0],
                                   np.array([119.4, 118.2, 116.6, 115.4, 114.4, 114.2]))
        np.testing.assert_allclose(resampled_cube.precipitation_max.values[..., 0, 0],
                                   np.array([120.0, 119.2, 118.0, 116.4, 115.2, 114.2]))

        schema = CubeSchema.new(resampled_cube)
        self.assertEqual(3, schema.ndim)
        self.assertEqual(('time', 'lat', 'lon'), schema.dims)
        self.assertEqual((6, 180, 360), schema.shape)
        self.assertEqual((1, 90, 180), schema.chunks)

    def test_resample_in_time_p90_dask(self):
        # "percentile_<p>" can currently only be used with numpy rather than chunked dask arrays:

        # TypeError raised on Windows, ValueError on Linux
        with self.assertRaises(Exception):
            # noinspection PyUnusedLocal
            resampled_cube = resample_in_time(self.input_cube, '2W', 'percentile_90')

    # TODO (forman): the call to resample_in_time() takes forever,
    #                this is not xcube, but may be an issue in dask 0.14 or dask 2.8.
    # def test_resample_in_time_p90_numpy(self):
    #     # "percentile_<p>" can currently only be used with numpy, so compute() first:
    #     input_cube = self.input_cube.compute()
    #     resampled_cube = resample_in_time(input_cube, '2W', 'percentile_90')
    #     self.assertIsNot(resampled_cube, self.input_cube)
    #     self.assertIn('time', resampled_cube)
    #     self.assertIn('temperature_p90', resampled_cube)
    #     self.assertIn('precipitation_p90', resampled_cube)
    #     self.assertEqual(('time',), resampled_cube.time.dims)
    #     self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_p90.dims)
    #     self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_p90.dims)
    #     self.assertEqual((6,), resampled_cube.time.shape)
    #     self.assertEqual((6, 180, 360), resampled_cube.temperature_p90.shape)
    #     self.assertEqual((6, 180, 360), resampled_cube.precipitation_p90.shape)
    #     np.testing.assert_equal(resampled_cube.time.values,
    #                             np.array(
    #                                 ['2017-06-25T00:00:00Z', '2017-07-09T00:00:00Z',
    #                                  '2017-07-23T00:00:00Z', '2017-08-06T00:00:00Z',
    #                                  '2017-08-20T00:00:00Z', '2017-09-03T00:00:00Z'], dtype=np.datetime64))
    #     np.testing.assert_allclose(resampled_cube.temperature_p90.values[..., 0, 0],
    #                                np.array([272.3, 272.9, 273.7, 274.3, 274.8, 274.9]))
    #     np.testing.assert_allclose(resampled_cube.precipitation_p90.values[..., 0, 0],
    #                                np.array([120.0, 119.2, 118.0, 116.4, 115.2, 114.2]))
    #
    #     schema = CubeSchema.new(resampled_cube)
    #     self.assertEqual(3, schema.ndim)
    #     self.assertEqual(('time', 'lat', 'lon'), schema.dims)
    #     self.assertEqual((6, 180, 360), schema.shape)
    #     self.assertEqual((1, 90, 180), schema.chunks)

    def test_resample_in_time_with_time_chunk_size(self):
        resampled_cube = resample_in_time(self.input_cube, '2D', ['min', 'max'], time_chunk_size=5)
        schema = CubeSchema.new(resampled_cube)
        self.assertEqual(3, schema.ndim)
        self.assertEqual(('time', 'lat', 'lon'), schema.dims)
        self.assertEqual((33, 180, 360), schema.shape)
        self.assertEqual((5, 90, 180), schema.chunks)

    def test_resample_f_all(self):
        resampled_cube = resample_in_time(self.input_cube, 'all', ['min', 'max'])
        self.assertIsNot(resampled_cube, self.input_cube)
        self.assertIn('time', resampled_cube)
        self.assertIn('temperature_min', resampled_cube)
        self.assertIn('temperature_max', resampled_cube)
        self.assertIn('precipitation_min', resampled_cube)
        self.assertIn('precipitation_max', resampled_cube)
        self.assertEqual(('time',), resampled_cube.time.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_min.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.temperature_max.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_min.dims)
        self.assertEqual(('time', 'lat', 'lon'), resampled_cube.precipitation_max.dims)
        self.assertEqual((1,), resampled_cube.time.shape)
        self.assertEqual((1, 180, 360), resampled_cube.temperature_min.shape)
        self.assertEqual((1, 180, 360), resampled_cube.temperature_max.shape)
        self.assertEqual((1, 180, 360), resampled_cube.precipitation_min.shape)
        self.assertEqual((1, 180, 360), resampled_cube.precipitation_max.shape)
        np.testing.assert_allclose(resampled_cube.temperature_min.values[..., 0, 0],
                                   np.array([272.0]))
        np.testing.assert_allclose(resampled_cube.temperature_max.values[..., 0, 0],
                                   np.array([274.9]))
        np.testing.assert_allclose(resampled_cube.precipitation_min.values[..., 0, 0],
                                   np.array([114.2]))
        np.testing.assert_allclose(resampled_cube.precipitation_max.values[..., 0, 0],
                                   np.array([120.0]))

        schema = CubeSchema.new(resampled_cube)
        self.assertEqual(3, schema.ndim)
        self.assertEqual(('time', 'lat', 'lon'), schema.dims)
        self.assertEqual((1, 180, 360), schema.shape)


# test/core/store/test_datatype.py

import copy
import unittest

import geopandas
import xarray

from xcube.core.mldataset import MultiLevelDataset
from xcube.core.store.datatype import ANY_TYPE, DataTypeLike
from xcube.core.store.datatype import DATASET_TYPE
from xcube.core.store.datatype import DataType
from xcube.core.store.datatype import GEO_DATA_FRAME_TYPE
from xcube.core.store.datatype import MULTI_LEVEL_DATASET_TYPE
from xcube.util.jsonschema import JsonStringSchema


class A:
    pass


class B(A):
    pass


class C:
    pass


class DataTypeTest(unittest.TestCase):
    def test_normalize_to_any(self):
        self.assertNormalizeOk('any',
                               ANY_TYPE,
                               object,
                               'any',
                               ('any', '*', 'object', 'builtins.object'))
        self.assertNormalizeOk(None,
                               ANY_TYPE,
                               object,
                               'any',
                               ('any', '*', 'object', 'builtins.object'))
        self.assertNormalizeOk(type(None),
                               ANY_TYPE,
                               object,
                               'any',
                               ('any', '*', 'object', 'builtins.object'))

    def test_normalize_to_dataset(self):
        self.assertNormalizeOk('dataset',
                               DATASET_TYPE,
                               xarray.Dataset,
                               'dataset',
                               ('dataset',
                                'xarray.Dataset',
                                'xarray.core.dataset.Dataset'))

    def test_normalize_to_mldataset(self):
        self.assertNormalizeOk('mldataset',
                               MULTI_LEVEL_DATASET_TYPE,
                               MultiLevelDataset,
                               'mldataset',
                               ('mldataset',
                                'xcube.MultiLevelDataset',
                                'xcube.core.mldataset.MultiLevelDataset'))

    def test_normalize_to_geodataframe(self):
        self.assertNormalizeOk('geodataframe',
                               GEO_DATA_FRAME_TYPE,
                               geopandas.GeoDataFrame,
                               'geodataframe',
                               ('geodataframe',
                                'geopandas.GeoDataFrame',
                                'geopandas.geodataframe.GeoDataFrame'))

    def assertNormalizeOk(self,
                          data_type: DataTypeLike,
                          expected_data_type,
                          expected_dtype,
                          expected_alias,
                          expected_aliases):
        data_type = DataType.normalize(data_type)
        self.assertIs(expected_data_type, data_type)
        self.assertIs(expected_dtype, data_type.dtype)
        self.assertEqual(expected_alias, data_type.alias)
        self.assertEqual(expected_alias, str(data_type))
        self.assertEqual(f'{expected_alias!r}', repr(data_type))
        self.assertEqual(expected_aliases, data_type.aliases)
        for other_alias in data_type.aliases:
            self.assertIs(data_type,
                          DataType.normalize(other_alias))
        self.assertIs(expected_data_type,
                      DataType.normalize(expected_dtype))
        self.assertIs(expected_data_type,
                      DataType.normalize(expected_data_type))

    def test_normalize_non_default_types(self):
        data_type = DataType.normalize(str)
        self.assertIs(str, data_type.dtype)
        self.assertEqual('builtins.str', data_type.alias)

    def test_normalize_failure(self):
        with self.assertRaises(ValueError) as cm:
            DataType.normalize('Gnartz')
        self.assertEqual("unknown data type 'Gnartz'",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            DataType.normalize(42)
        self.assertEqual("cannot convert 42 into a data type",
                         f'{cm.exception}')

    def test_equality(self):
        self.assertIs(DATASET_TYPE, DATASET_TYPE)
        self.assertEqual(DATASET_TYPE, DATASET_TYPE)

        self.assertEqual(copy.deepcopy(DATASET_TYPE), DATASET_TYPE)
        self.assertNotEqual(MULTI_LEVEL_DATASET_TYPE, DATASET_TYPE)

        self.assertEqual(hash(copy.deepcopy(DATASET_TYPE)), hash(DATASET_TYPE))
        self.assertNotEqual(hash(MULTI_LEVEL_DATASET_TYPE), hash(DATASET_TYPE))

    def test_is_sub_type_of(self):
        self.assertTrue(ANY_TYPE.is_sub_type_of(ANY_TYPE))
        self.assertFalse(ANY_TYPE.is_sub_type_of(DATASET_TYPE))
        self.assertFalse(ANY_TYPE.is_sub_type_of(MULTI_LEVEL_DATASET_TYPE))

        self.assertTrue(DATASET_TYPE.is_sub_type_of(ANY_TYPE))
        self.assertTrue(DATASET_TYPE.is_sub_type_of(DATASET_TYPE))
        self.assertFalse(DATASET_TYPE.is_sub_type_of(MULTI_LEVEL_DATASET_TYPE))

        a_type = DataType.normalize(A)
        b_type = DataType.normalize(B)
        c_type = DataType.normalize(C)
        self.assertTrue(a_type.is_sub_type_of(a_type))
        self.assertFalse(a_type.is_sub_type_of(b_type))
        self.assertFalse(a_type.is_sub_type_of(c_type))
        self.assertTrue(b_type.is_sub_type_of(a_type))
        self.assertTrue(b_type.is_sub_type_of(b_type))
        self.assertFalse(b_type.is_sub_type_of(c_type))
        self.assertFalse(c_type.is_sub_type_of(a_type))
        self.assertFalse(c_type.is_sub_type_of(b_type))
        self.assertTrue(c_type.is_sub_type_of(c_type))

    def test_is_super_type_of(self):
        self.assertTrue(ANY_TYPE.is_super_type_of(ANY_TYPE))
        self.assertTrue(ANY_TYPE.is_super_type_of(DATASET_TYPE))
        self.assertTrue(ANY_TYPE.is_super_type_of(DATASET_TYPE))

        self.assertTrue(ANY_TYPE.is_super_type_of(DATASET_TYPE))
        self.assertTrue(DATASET_TYPE.is_super_type_of(DATASET_TYPE))
        self.assertFalse(MULTI_LEVEL_DATASET_TYPE.is_super_type_of(DATASET_TYPE))

        a_type = DataType.normalize(A)
        b_type = DataType.normalize(B)
        c_type = DataType.normalize(C)
        self.assertTrue(a_type.is_super_type_of(a_type))
        self.assertTrue(a_type.is_super_type_of(b_type))
        self.assertFalse(a_type.is_super_type_of(c_type))
        self.assertFalse(b_type.is_super_type_of(a_type))
        self.assertTrue(b_type.is_super_type_of(b_type))
        self.assertFalse(b_type.is_super_type_of(c_type))
        self.assertFalse(c_type.is_super_type_of(a_type))
        self.assertFalse(c_type.is_super_type_of(b_type))
        self.assertTrue(c_type.is_super_type_of(c_type))

    def test_schema(self):
        self.assertIsInstance(DATASET_TYPE.get_schema(), JsonStringSchema)


# test/core/store/test_descriptor.py

import unittest
from typing import Type

import jsonschema
import numpy as np

from xcube.core.mldataset import BaseMultiLevelDataset
from xcube.core.new import new_cube
from xcube.core.store.datatype import DataType
from xcube.core.store.descriptor import DataDescriptor, _attrs_to_json, MultiLevelDatasetDescriptor
from xcube.core.store.descriptor import DatasetDescriptor
from xcube.core.store.descriptor import GeoDataFrameDescriptor
from xcube.core.store.descriptor import VariableDescriptor
from xcube.core.store.descriptor import new_data_descriptor
from xcube.util.jsonschema import JsonBooleanSchema
from xcube.util.jsonschema import JsonObjectSchema


class NewDataDescriptorTest(unittest.TestCase):

    def test_new_dataset_descriptor(self):
        cube = new_cube(variables=dict(a=4.1, b=7.4))
        descriptor = new_data_descriptor('cube', cube)
        self.assertExpectedDescriptor(descriptor,
                                      DatasetDescriptor,
                                      'dataset')

    def test_new_dataset_descriptor_with_bbox_from_metadata(self):
        cube = new_cube(
            variables=dict(a=4.1, b=7.4),
            x_name='you_did',
            y_name='not_expect_this'
        )
        cube.attrs.update(geospatial_lon_min=50,
                          geospatial_lon_max=60,
                          geospatial_lat_max=40,
                          geospatial_lat_min=30)
        cube.you_did.attrs.update(standard_name='you_did', long_name='you_did')
        cube.not_expect_this.attrs.update(standard_name='not_expect_this',
                                          long_name='not_expect_this')
        descriptor = new_data_descriptor('cube', cube)
        self.assertEqual((50, 30, 60, 40), descriptor.bbox)

    def test_new_ml_dataset_descriptor(self):
        cube = new_cube(variables=dict(a=4.1, b=7.4))
        ml_cube = BaseMultiLevelDataset(cube)
        descriptor = new_data_descriptor('cube', ml_cube)
        self.assertExpectedDescriptor(descriptor,
                                      MultiLevelDatasetDescriptor,
                                      'mldataset')

    def assertExpectedDescriptor(self,
                                 descriptor: DataDescriptor,
                                 expected_type: Type[DataDescriptor],
                                 expected_data_type_alias: str):
        self.assertIsInstance(descriptor, DatasetDescriptor)
        self.assertIsInstance(descriptor, expected_type)
        self.assertIsInstance(descriptor.data_type, DataType)
        self.assertEqual(expected_data_type_alias, descriptor.data_type.alias)
        self.assertEqual('cube', descriptor.data_id)
        self.assertEqual((-180.0, -90.0, 180.0, 90.0), descriptor.bbox)
        self.assertIsNone(descriptor.open_params_schema)
        self.assertEqual(('2010-01-01', '2010-01-06'), descriptor.time_range)
        self.assertEqual('1D', descriptor.time_period)
        self.assertEqual(1.0, descriptor.spatial_res)
        self.assertIsNotNone(descriptor.coords)
        self.assertEqual({'time': 5, 'lat': 180, 'lon': 360, 'bnds': 2}, descriptor.dims)
        self.assertIsNotNone(descriptor.data_vars)


class DataDescriptorTest(unittest.TestCase):

    def test_from_dict_no_data_id(self):
        with self.assertRaises(jsonschema.exceptions.ValidationError):
            descriptor_dict = dict()
            DataDescriptor.from_dict(descriptor_dict)

    def test_from_dict_no_data_type(self):
        with self.assertRaises(jsonschema.exceptions.ValidationError):
            descriptor_dict = dict(data_id='id')
            DataDescriptor.from_dict(descriptor_dict)

    def test_from_dict_unknown_data_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='tsr')
        with self.assertRaises(ValueError) as cm:
            DataDescriptor.from_dict(descriptor_dict)
        self.assertEqual("unknown data type 'tsr'", f'{cm.exception}')

    def test_from_dict_dataset_data_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='dataset')
        descriptor = DataDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertTrue(DatasetDescriptor, type(descriptor))
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)

    def test_from_dict_geodataframe_data_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='geodataframe')
        descriptor = DataDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertTrue(GeoDataFrameDescriptor, type(descriptor))
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('geodataframe', descriptor.data_type.alias)

    def test_from_dict_full(self):
        descriptor_dict = dict(
            data_id='CHL.zarr',
            data_type='dataset',
            crs='EPSG:4326',
            bbox=(10., 20., 30., 40.),
            time_range=('2017-06-05', '2017-06-27'),
            time_period='2D',
            open_params_schema=dict(
                type="object",
                properties=dict(
                    variable_names=dict(
                        type='array',
                        items=dict(type='string')
                    )
                )
            )
        )
        descriptor = DataDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertEqual('CHL.zarr', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)
        self.assertEqual('EPSG:4326', descriptor.crs)
        self.assertEqual((10., 20., 30., 40.), descriptor.bbox)
        self.assertEqual(('2017-06-05', '2017-06-27'), descriptor.time_range)
        self.assertEqual('2D', descriptor.time_period)
        self.assertEqual('object',
                         descriptor.open_params_schema.get('type', None))

    def test_to_dict(self):
        descriptor = DatasetDescriptor(
            data_id='xyz',
            crs='EPSG:9346',
            bbox=(10., 20., 30., 40.),
            spatial_res=20.,
            time_range=('2017-06-05', '2017-06-27'),
            time_period='daily',
            open_params_schema=JsonObjectSchema(
                properties=dict(
                    consolidated=JsonBooleanSchema(),
                ),
                additional_properties=False,
            )
        )
        descriptor_dict = descriptor.to_dict()
        self.assertEqual(
            {
                'data_id': 'xyz',
                'crs': 'EPSG:9346',
                'data_type': 'dataset',
                'bbox': [10.0, 20.0, 30.0, 40.0],
                'spatial_res': 20.0,
                'time_range': ['2017-06-05', '2017-06-27'],
                'time_period': 'daily',
                'open_params_schema': {
                    'type': 'object',
                    'properties': {
                        'consolidated': {
                            'type': 'boolean'
                        }
                    },
                    'additionalProperties': False,
                },
            },
            descriptor_dict)


class DatasetDescriptorTest(unittest.TestCase):

    def test_get_schema(self):
        schema = DatasetDescriptor.get_schema()
        self.assertIsInstance(schema, JsonObjectSchema)

    def test_from_dict_no_data_id(self):
        descriptor_dict = dict()
        with self.assertRaises(jsonschema.exceptions.ValidationError):
            DatasetDescriptor.from_dict(descriptor_dict)

    def test_from_dict_wrong_data_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='tsr')
        with self.assertRaises(ValueError) as cm:
            DatasetDescriptor.from_dict(descriptor_dict)
        self.assertEqual("unknown data type 'tsr'",
                         f'{cm.exception}')

    def test_from_dict_derived_type(self):
        descriptor_dict = dict(data_id='xyz', data_type='dataset')
        descriptor = DatasetDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)

    def test_from_dict_full(self):
        descriptor_dict = dict(
            data_id='xyz',
            data_type='dataset',
            crs='EPSG:9346',
            bbox=(10., 20., 30., 40.),
            spatial_res=20.,
            time_range=('2017-06-05', '2017-06-27'),
            time_period='daily',
            coords=dict(
                rtdt=dict(
                    name='rtdt',
                    dtype='rj',
                    dims=('rtdt',),
                    attrs=dict(
                        ssd=6,
                        zjgrhgu='hgtr'
                    )
                )
            ),
            dims=dict(x=1, y=2, z=3),
            data_vars=dict(
                xf=dict(
                    name='xf',
                    dtype='rj',
                    dims=('dfjhrt', 'sg'),
                    chunks=(2, 3),
                    attrs=dict(
                        ssd=4,
                        zjgrhgu='dgfrf'
                    )
                )
            ),
            attrs=dict(
                dzus=236,
                tgr7h='rt5',
                df='s8fd4w5'
            ),
            open_params_schema=dict(
                type="object",
                properties=dict(
                    variable_names=dict(
                        type='array',
                        items=dict(
                            type='string'
                        )
                    )
                )
            )
        )
        descriptor = DatasetDescriptor.from_dict(descriptor_dict)
        self.assertIsNotNone(descriptor)
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)
        self.assertEqual('EPSG:9346', descriptor.crs)
        self.assertEqual((10., 20., 30., 40.), descriptor.bbox)
        self.assertEqual(20., descriptor.spatial_res)
        self.assertEqual(('2017-06-05', '2017-06-27'), descriptor.time_range)
        self.assertEqual('daily', descriptor.time_period)
        self.assertEqual(1, len(descriptor.coords))
        self.assertTrue('rtdt' in descriptor.coords)
        self.assertEqual(dict(x=1, y=2, z=3), descriptor.dims)
        self.assertEqual(1, len(descriptor.data_vars))
        self.assertTrue('xf' in descriptor.data_vars)
        self.assertEqual(236, descriptor.attrs.get('dzus', None))
        self.assertEqual('rt5', descriptor.attrs.get('tgr7h', None))
        self.assertEqual('s8fd4w5', descriptor.attrs.get('df', None))
        self.assertEqual('object', descriptor.open_params_schema.get('type', None))

    def test_from_dict_var_descriptors_as_dict(self):
        descriptor_dict = dict(
            data_id='xyz',
            data_type='dataset',
            crs='EPSG:4326',
            data_vars=dict(
                A=dict(
                    name='A',
                    dtype='float32',
                    dims=('time', 'lat', 'lon'),
                    chunks=(2, 3),
                    attrs=dict(
                        ssd=4,
                        zjgrhgu='dgfrf'
                    )
                )
            )
        )
        descriptor = DatasetDescriptor.from_dict(descriptor_dict)
        self.assertEqual('xyz', descriptor.data_id)
        self.assertEqual('dataset', descriptor.data_type.alias)
        self.assertEqual('EPSG:4326', descriptor.crs)
        self.assertEqual(1, len(descriptor.data_vars))
        self.assertTrue('A' in descriptor.data_vars)
        self.assertIsInstance(descriptor.data_vars.get('A'),
                              VariableDescriptor)

    def test_to_dict(self):
        coords = dict(
            rtdt=VariableDescriptor(
                name='rtdt',
                dtype='rj',
                dims=('rtdt',),
                chunks=(2,),
                attrs=dict(
                    ssd=6,
                    zjgrhgu='hgtr'
                )
            )
        )
        var_descriptors = dict(
            xf=VariableDescriptor(
                name='xf',
                dtype='rj',
                dims=('dfjhrt', 'sg'),
                chunks=(2, 3),
                attrs=dict(
                    ssd=4,
                    zjgrhgu='dgfrf'
                )
            )
        )
        descriptor = DatasetDescriptor(
            data_id='xyz',
            crs='EPSG:9346',
            bbox=(10., 20., 30., 40.),
            spatial_res=20.,
            time_range=('2017-06-05', '2017-06-27'),
            time_period='daily',
            coords=coords,
            dims=dict(
                x=1, y=2, z=3
            ),
            data_vars=var_descriptors,
            attrs=dict(
                dzus=236,
                tgr7h='rt5',
                df='s8fd4w5'
            )
        )
        descriptor_dict = descriptor.to_dict()
        self.assertEqual(
            dict(
                data_id='xyz',
                data_type='dataset',
                crs='EPSG:9346',
                bbox=[10., 20., 30., 40.],
                spatial_res=20.,
                time_range=['2017-06-05', '2017-06-27'],
                time_period='daily',
                coords=dict(
                    rtdt=dict(
                        name='rtdt',
                        dtype='rj',
                        dims=['rtdt', ],
                        chunks=[2, ],
                        attrs=dict(
                            ssd=6,
                            zjgrhgu='hgtr'
                        )
                    )
                ),
                dims=dict(x=1, y=2, z=3),
                data_vars=dict(
                    xf=dict(
                        name='xf',
                        dtype='rj',
                        dims=['dfjhrt', 'sg'],
                        chunks=[2, 3],
                        attrs=dict(
                            ssd=4,
                            zjgrhgu='dgfrf'
                        )
                    )
                ),
                attrs=dict(
                    dzus=236,
                    tgr7h='rt5',
                    df='s8fd4w5'
                )
            ),
            descriptor_dict
        )


class VariableDescriptorTest(unittest.TestCase):

    def test_variable_descriptor_basic(self):
        vd1 = VariableDescriptor('gz',
                                 'zughysz',
                                 ['rtdswgt', 'dref', 'zdrs5ge'],
                                 chunks=(3, 321, 4))
        self.assertEqual('gz', vd1.name)
        self.assertEqual('zughysz', vd1.dtype)
        self.assertEqual(('rtdswgt', 'dref', 'zdrs5ge'), vd1.dims)
        self.assertEqual(3, vd1.ndim)
        self.assertEqual((3, 321, 4), vd1.chunks)
        self.assertEqual(None, vd1.attrs)

        vd3 = VariableDescriptor('gz',
                                 'zughysz',
                                 ['rtdswgt', 'dref', 'zdrs5ge'],
                                 chunks=(3, 321, 4),
                                 attrs={'d': 2, 'zjgu': ''})
        self.assertEqual('gz', vd3.name)
        self.assertEqual('zughysz', vd3.dtype)
        self.assertEqual(('rtdswgt', 'dref', 'zdrs5ge'), vd3.dims)
        self.assertEqual(3, vd3.ndim)
        self.assertEqual((3, 321, 4), vd3.chunks)
        self.assertEqual({'d': 2, 'zjgu': ''}, vd3.attrs)

    def test_variable_descriptor_to_dict(self):
        vd = VariableDescriptor('xf',
                                'rj',
                                ['dfjhrt', 'sg'],
                                chunks=(3, 2),
                                attrs={'ssd': 4, 'zjgrhgu': 'dgfrf', 'fill_value': np.NaN})
        expected = {
            'name': 'xf',
            'dtype': 'rj',
            'dims': ['dfjhrt', 'sg'],
            'chunks': [3, 2],
            'attrs': {
                'ssd': 4,
                'zjgrhgu': 'dgfrf',
                'fill_value': None
            }
        }
        self.assertEqual(expected, vd.to_dict())

    def test_variable_descriptor_from_dict(self):
        vd_as_dict = {
            'name': 'xf',
            'dtype': 'rj',
            'dims': ('dfjhrt', 'sg'),
            'chunks': (3, 2),
            'attrs': {
                'ssd': 4,
                'zjgrhgu': 'dgfrf',
                'fill_value': None
            }
        }
        vd = VariableDescriptor.from_dict(vd_as_dict)
        self.assertEqual('xf', vd.name)
        self.assertEqual('rj', vd.dtype)
        self.assertEqual(('dfjhrt', 'sg'), vd.dims)
        self.assertEqual(2, vd.ndim)
        self.assertEqual((3, 2), vd.chunks)
        self.assertEqual({'ssd': 4, 'zjgrhgu': 'dgfrf', 'fill_value': None}, vd.attrs)

        with self.assertRaises(jsonschema.exceptions.ValidationError):
            VariableDescriptor.from_dict({'name': 'dhgfr', 'dtype': 'ghdst'})

        with self.assertRaises(jsonschema.exceptions.ValidationError):
            VariableDescriptor.from_dict({'name': 'dhgfr', 'dims': ['faer', 'bjunda']})

        with self.assertRaises(jsonschema.exceptions.ValidationError):
            VariableDescriptor.from_dict({'dtype': 'ghdst', 'dims': ['faer', 'bjunda']})


class JsonTest(unittest.TestCase):
    def test_attrs_to_json_numpy(self):
        self.assertEqual(
            {
                "num_bands": 17,
                "_FillValue": None,
                "flag_names": "F1 F2 F3 F4 F5",
                "flag_values": [1, 2, 4, 8, 16]
            },
            _attrs_to_json(
                {
                    "num_bands": np.array(17),
                    "_FillValue": np.nan,
                    "flag_names": "F1 F2 F3 F4 F5",
                    "flag_values": np.array([1, 2, 4, 8, 16])
                }
            )
        )

    def test_attrs_to_json_dask(self):
        import dask.array as da
        self.assertEqual(
            {
                "flag_names": "F1 F2 F3 F4 F5",
                "flag_values": [1, 2, 4, 8, 16]
            },
            _attrs_to_json(
                {
                    "flag_names": "F1 F2 F3 F4 F5",
                    "flag_values": da.from_array([1, 2, 4, 8, 16])
                }
            )
        )


# test/core/store/test_storepool.py

import json
import os
import unittest

import jsonschema
import yaml

from xcube.core.store import DataStore
from xcube.core.store import DataStoreConfig
from xcube.core.store import DataStoreError
from xcube.core.store import DataStoreInstance
from xcube.core.store import DataStorePool
from xcube.core.store import get_data_store_instance


class GetDataStoreTest(unittest.TestCase):

    def test_get_data_store_instance_new_inst(self):
        instance = get_data_store_instance('memory')
        self.assertIsInstance(instance, DataStoreInstance)
        self.assertIsInstance(instance.store, DataStore)
        instance2 = get_data_store_instance('memory')
        self.assertIsNot(instance, instance2)
        self.assertIsNot(instance.store, instance2.store)

    def test_get_data_store_instance_from_pool(self):
        pool = DataStorePool({
            'dir': DataStoreConfig('file',
                                   store_params=dict(root='.'))
        })
        instance = get_data_store_instance('@dir', store_pool=pool)
        self.assertTrue(hasattr(instance.store, 'root'))
        # noinspection PyUnresolvedReferences
        self.assertTrue(os.path.isabs(instance.store.root))
        self.assertTrue(os.path.isdir(instance.store.root))
        instance2 = get_data_store_instance('@dir', store_pool=pool)
        self.assertIs(instance, instance2)

    def test_get_data_store_instance_from_pool_with_params(self):
        pool = DataStorePool({
            '@dir': DataStoreConfig('file',
                                    store_params=dict(root='.'))
        })
        with self.assertRaises(ValueError) as cm:
            get_data_store_instance(
                '@dir', store_pool=pool, store_params={'auto_mkdir': True}
            )
        self.assertEqual('store_params cannot be given,'
                         ' with store_id ("@dir") referring'
                         ' to a configured store',
                         f'{cm.exception}')

    def test_get_data_store_instance_from_pool_without_pool(self):
        with self.assertRaises(ValueError) as cm:
            get_data_store_instance('@dir')
        self.assertEqual('store_pool must be given,'
                         ' with store_id ("@dir") referring'
                         ' to a configured store',
                         f'{cm.exception}')

    def test_normalize(self):
        pool = DataStorePool({
            '@dir': DataStoreConfig('directory',
                                    store_params=dict(root='.'))
        })
        file_path = '_test-data-stores-pool.json'
        with open(file_path, 'w') as fp:
            json.dump(pool.to_dict(), fp)
        try:
            pool_1 = DataStorePool.normalize(file_path)
            self.assertIsInstance(pool_1, DataStorePool)
            pool_2 = DataStorePool.normalize(pool_1)
            self.assertIs(pool_2, pool_1)
            pool_3 = DataStorePool.normalize(pool_2.to_dict())
            self.assertIsInstance(pool_3, DataStorePool)
        finally:
            os.remove(file_path)

        with self.assertRaises(TypeError):
            # noinspection PyTypeChecker
            DataStorePool.normalize(42)


class DataStoreConfigTest(unittest.TestCase):

    def test_constructor_and_instance_props(self):
        store_config = DataStoreConfig('file',
                                       store_params={'root': '.'},
                                       title='Local',
                                       description='Local files')
        self.assertEqual('file', store_config.store_id)
        self.assertEqual({'root': '.'}, store_config.store_params)
        self.assertEqual('Local', store_config.title)
        self.assertEqual('Local files', store_config.description)

    def test_constructor_asserts(self):
        with self.assertRaises(ValueError) as cm:
            DataStoreConfig('')
        self.assertEqual('store_id must be given', f'{cm.exception}')

        with self.assertRaises(TypeError) as cm:
            # noinspection PyTypeChecker
            DataStoreConfig('directory', store_params=[1, 'B'])
        self.assertEqual("store_params must be an instance"
                         " of <class 'dict'>, was <class 'list'>",
                         f'{cm.exception}')

    def test_to_dict(self):
        store_config = DataStoreConfig('directory',
                                       store_params={'base_dir': '.'},
                                       title='Local',
                                       description='Local files')
        self.assertEqual({'description': 'Local files',
                          'name': 'Local',
                          'store_id': 'directory',
                          'store_params': {'base_dir': '.'}},
                         store_config.to_dict())

    def test_from_dict(self):
        store_config = DataStoreConfig.from_dict({
            'description': 'Local files',
            'title': 'Local',
            'store_id': 'file',
            'store_params': {'root': '.'}
        })
        self.assertIsInstance(store_config, DataStoreConfig)
        self.assertEqual('file', store_config.store_id)
        self.assertEqual({'root': '.'}, store_config.store_params)
        self.assertEqual('Local', store_config.title)
        self.assertEqual('Local files', store_config.description)

    def test_from_dict_with_valid_cost_params(self):
        store_config = DataStoreConfig.from_dict({
            'description': 'Local files',
            'title': 'Local',
            'store_id': 'file',
            'store_params': {'root': '.'},
            'cost_params': {
                'input_pixels_per_punit': 500,
                'output_pixels_per_punit': 100,
                'input_punits_weight': 1.1,
            }
        })
        self.assertIsInstance(store_config, DataStoreConfig)
        self.assertEqual('file', store_config.store_id)
        self.assertEqual({'root': '.'}, store_config.store_params)
        self.assertEqual('Local', store_config.title)
        self.assertEqual('Local files', store_config.description)

    def test_from_dict_with_invalid_cost_params(self):
        with self.assertRaises(DataStoreError):
            DataStoreConfig.from_dict({'description': 'Local files',
                                       'title': 'Local',
                                       'store_id': 'file',
                                       'store_params': {'root': '.'},
                                       'cost_params': {
                                           # Required:
                                           # 'input_pixels_per_punit': 10,
                                           # 'output_pixels_per_punit': 20,
                                       }})


class DataStorePoolTest(unittest.TestCase):
    def test_default_constr(self):
        pool = DataStorePool()
        self.assertEqual([], pool.store_instance_ids)
        self.assertEqual([], pool.store_configs)

    def test_from_dict_empty(self):
        pool = DataStorePool.from_dict({})
        self.assertIsInstance(pool, DataStorePool)
        self.assertEqual([], pool.store_instance_ids)
        self.assertEqual([], pool.store_configs)

    def test_from_dict_no_store_params(self):
        store_configs = {
            "ram-1": {
                "store_id": "memory"
            }
        }
        pool = DataStorePool.from_dict(store_configs)
        self.assertIsInstance(pool, DataStorePool)
        self.assertEqual(["ram-1"], pool.store_instance_ids)
        self.assertIsInstance(pool.get_store_config('ram-1'), DataStoreConfig)

    def test_from_dict_with_bad_dicts(self):
        store_configs = {
            "dir": {
            }
        }
        with self.assertRaises(jsonschema.exceptions.ValidationError) as cm:
            DataStorePool.from_dict(store_configs)
        self.assertTrue(
            "'store_id' is a required property" in f'{cm.exception}',
            msg=f'{cm.exception}')

        store_configs = {
            "dir": {
                "store_id": 10
            }
        }
        with self.assertRaises(jsonschema.exceptions.ValidationError) as cm:
            DataStorePool.from_dict(store_configs)
        self.assertTrue(
            "Failed validating 'type' in schema" in f'{cm.exception}',
            msg=f'{cm.exception}')

    def test_from_json_file(self):
        self._assert_file_ok('json')

    def test_from_yaml_file(self):
        self._assert_file_ok('yaml')

    def test_from_json_file_env(self):
        self._assert_file_ok('json', use_env_vars=True)
        self._assert_file_ok('json',
                             root_1='/test_1',
                             root_2='/test_2',
                             use_env_vars=True)

    def test_from_yaml_file_env(self):
        self._assert_file_ok('yaml', use_env_vars=True)
        self._assert_file_ok('yaml',
                             root_1='/test_1',
                             root_2='/test_2',
                             use_env_vars=True)

    def _assert_file_ok(self,
                        format_name: str,
                        root_1="/root1",
                        root_2="/root2",
                        use_env_vars=False):
        if use_env_vars:
            store_configs = self._get_test_config(
                root_1='${_TEST_ROOT_1}',
                root_2='${_TEST_ROOT_2}'
            )
            import os
            os.environ['_TEST_ROOT_1'] = root_1
            os.environ['_TEST_ROOT_2'] = root_2
        else:
            store_configs = self._get_test_config(
                root_1=root_1,
                root_2=root_2
            )
        path = 'test-store-configs.' + format_name
        with open(path, 'w') as fp:
            mod = yaml if format_name == 'yaml' else json
            mod.dump(store_configs, fp, indent=2)
        try:
            pool = DataStorePool.from_file(path)
            self.assertIsInstance(pool, DataStorePool)
            self.assertEqual(['ram-1', 'ram-2'], pool.store_instance_ids)
            config_1 = pool.get_store_config('ram-1')
            self.assertIsInstance(config_1, DataStoreConfig)
            self.assertEqual(
                {'store_id': 'memory',
                 'store_params': {'root': root_1}},
                config_1.to_dict())
            config_2 = pool.get_store_config('ram-2')
            self.assertIsInstance(config_2, DataStoreConfig)
            self.assertEqual(
                {'store_id': 'memory',
                 'store_params': {'root': root_2}},
                config_2.to_dict())
        finally:
            import os
            os.remove(path)

    @staticmethod
    def _get_test_config(root_1: str, root_2: str):
        return {
            "ram-1": {
                "store_id": "memory",
                "store_params": {
                    "root": root_1
                },
            },
            "ram-2": {
                "store_id": "memory",
                "store_params": {
                    "root": root_2
                },
            }
        }

    def test_get_store(self):
        store_configs = {
            "dir-1": {
                "store_id": "file",
                "store_params": {
                    "root": "./bibo"
                }
            },
        }
        pool = DataStorePool.from_dict(store_configs)
        store = pool.get_store('dir-1')
        self.assertTrue(hasattr(store, 'root'))
        # noinspection PyUnresolvedReferences
        self.assertTrue(os.path.isabs(store.root))
        self.assertFalse(os.path.exists(store.root))
        # Should stay same instance
        self.assertIs(store, pool.get_store('dir-1'))
        self.assertIs(store, pool.get_store('dir-1'))

    def test_get_store_error(self):
        pool = DataStorePool()
        with self.assertRaises(DataStoreError) as cm:
            pool.get_store('dir-1')
        self.assertEqual('Configured data store instance "dir-1" not found.', f'{cm.exception}')

    def test_to_dict(self):
        self.assertEqual({}, DataStorePool().to_dict())
        self.assertEqual({'ram': {'store_id': 'memory'},
                          'dir': {'store_id': 'file',
                                  'store_params': {'base_dir': 'bibo'}}},
                         DataStorePool({'ram': DataStoreConfig(store_id='memory'),
                                        'dir': DataStoreConfig(store_id='file',
                                                               store_params=dict(base_dir="bibo"))
                                        }).to_dict())

    def test_close_all_stores(self):
        store_configs = {
            "ram-1": {
                "store_id": "memory",
            },
        }
        pool = DataStorePool.from_dict(store_configs)
        # Smoke test, we do not expect any visible state changes after close_all_stores()
        pool.close_all_stores()

    def test_add_remove_store_config(self):
        pool = DataStorePool()
        self.assertEqual([], pool.store_instance_ids)
        pool.add_store_config('mem-1', DataStoreConfig('memory'))
        self.assertEqual(['mem-1'], pool.store_instance_ids)
        pool.add_store_config('mem-2', DataStoreConfig('memory'))
        self.assertEqual(['mem-1', 'mem-2'], pool.store_instance_ids)
        pool.add_store_config('mem-1', DataStoreConfig('memory'))
        self.assertEqual(['mem-1', 'mem-2'], pool.store_instance_ids)
        pool.remove_store_config('mem-1')
        self.assertEqual(['mem-2'], pool.store_instance_ids)
        pool.remove_store_config('mem-2')
        self.assertEqual([], pool.store_instance_ids)

    def test_multi_stores_with_params(self):
        """Just test many stores at once"""
        store_configs = {
            "ram-1": {
                "store_id": "memory",
            },
            "ram-2": {
                "store_id": "memory",
            },
            "local-1": {
                "store_id": "file",
                "store_params": {
                    "root": "/home/bibo/datacubes-1",
                }
            },
            "local-2": {
                "store_id": "file",
                "store_params": {
                    "root": "/home/bibo/datacubes-2",
                }
            },
        }
        pool = DataStorePool.from_dict(store_configs)
        self.assertIsInstance(pool, DataStorePool)
        self.assertEqual(["local-1", "local-2", "ram-1", "ram-2"], pool.store_instance_ids)
        for instance_id in pool.store_instance_ids:
            self.assertTrue(pool.has_store_instance(instance_id))
            self.assertIsInstance(pool.get_store_config(instance_id), DataStoreConfig)
            self.assertIsInstance(pool.get_store(instance_id), DataStore)

    def test_get_store_instance_id(self):
        store_params_1 = {
            "root": "./bibo"
        }
        ds_config_1 = DataStoreConfig(store_id='file',
                                      store_params=store_params_1)
        ds_configs = {'dir-1': ds_config_1}
        pool = DataStorePool(ds_configs)

        store_params_2 = {
            "root": "./babo"
        }
        ds_config_2 = DataStoreConfig(store_id='file',
                                      store_params=store_params_2)
        ds_config_3 = DataStoreConfig(store_id='file',
                                      store_params=store_params_1,
                                      title='A third configuration')

        self.assertEqual('dir-1', pool.get_store_instance_id(ds_config_1))
        self.assertEqual('dir-1', pool.get_store_instance_id(ds_config_1,
                                                             strict_check=True))

        self.assertIsNone(pool.get_store_instance_id(ds_config_2))

        self.assertEqual('dir-1', pool.get_store_instance_id(ds_config_3))
        self.assertIsNone(pool.get_store_instance_id(ds_config_3,
                                                     strict_check=True))


# test/server/webservers/__init__.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.


# test/server/webservers/test_flask.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest

from xcube.server.webservers.flask import FlaskFramework


class TornadoFrameworkTest(unittest.TestCase):

    def setUp(self) -> None:
        self.framework = FlaskFramework()

    def test_add_routes(self):
        with self.assertRaises(NotImplementedError):
            self.framework.add_routes([])

    def test_start(self):
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            self.framework.start(None)

    def test_stop(self):
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            self.framework.stop(None)

    def test_update(self):
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            self.framework.update(None)

    def test_call_later(self):
        def my_func():
            pass

        with self.assertRaises(NotImplementedError):
            self.framework.call_later(0.1, my_func)

    def test_run_in_executor(self):
        def my_func():
            pass

        with self.assertRaises(NotImplementedError):
            self.framework.run_in_executor(None, my_func)


# test/server/webservers/test_tornado.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest
from typing import Sequence, Optional, Callable, Any, Awaitable, Union, Tuple, \
    Type, Dict

import pytest
import tornado.httputil
import tornado.web
from tornado import concurrent

from test.server.mocks import mock_server
from xcube.server.api import Api
from xcube.server.api import ApiContextT
from xcube.server.api import ApiError
from xcube.server.api import ApiHandler
from xcube.server.api import ApiRoute
from xcube.server.api import Context
from xcube.server.api import ServerConfig
from xcube.server.asyncexec import ReturnT
from xcube.server.webservers.tornado import SERVER_CTX_ATTR_NAME
from xcube.server.webservers.tornado import TornadoApiRequest
from xcube.server.webservers.tornado import TornadoFramework
from xcube.server.webservers.tornado import TornadoRequestHandler


class TornadoFrameworkTest(unittest.TestCase):
    def setUp(self) -> None:
        self.application = MockApplication()
        self.io_loop = MockIOLoop()
        # noinspection PyTypeChecker
        self.framework = TornadoFramework(application=self.application,
                                          io_loop=self.io_loop)

    # def test_add_routes(self):
    #     self.framework.add_routes([ApiRoute()])

    def test_start_and_update_and_stop(self):
        server = mock_server(framework=self.framework, config={})
        self.assertEqual(0, self.application.listen_count)
        self.assertEqual(0, self.io_loop.start_count)
        self.assertEqual(0, self.io_loop.stop_count)
        self.framework.start(server.ctx)
        self.assertEqual(1, self.application.listen_count)
        self.assertEqual(1, self.io_loop.start_count)
        self.assertEqual(0, self.io_loop.stop_count)
        self.framework.update(server.ctx)
        self.assertEqual(1, self.application.listen_count)
        self.assertEqual(1, self.io_loop.start_count)
        self.assertEqual(0, self.io_loop.stop_count)
        self.framework.stop(server.ctx)
        self.assertEqual(1, self.application.listen_count)
        self.assertEqual(1, self.io_loop.start_count)
        self.assertEqual(1, self.io_loop.stop_count)

    def test_async_exec(self):
        def my_func(a, b):
            return a + b

        self.assertEqual(0, self.io_loop.call_later_count)
        self.framework.call_later(0.1, my_func, 40, 2)
        self.assertEqual(1, self.io_loop.call_later_count)

        self.assertEqual(0, self.io_loop.run_in_executor_count)
        self.framework.run_in_executor(None, my_func, 40, 2)
        self.assertEqual(1, self.io_loop.run_in_executor_count)

    def test_path_to_pattern_ok(self):
        p2p = TornadoFramework.path_to_pattern

        self.assertEqual("/collections",
                         p2p("/collections"))

        self.assertEqual("/collections/"
                         "("
                         "?P<collection_id>"
                         "[^\\;\\/\\?\\:\\@\\&\\=\\+\\$\\,]+"
                         ")",
                         p2p("/collections/{collection_id}"))

        self.assertEqual("/collections/"
                         "("
                         "?P<collection_id>"
                         "[^\\;\\/\\?\\:\\@\\&\\=\\+\\$\\,]+"
                         ")"
                         "/items",
                         p2p("/collections/{collection_id}/items"))

        self.assertEqual("/collections/"
                         "("
                         "?P<collection_id>"
                         "[^\\;\\/\\?\\:\\@\\&\\=\\+\\$\\,]+"
                         ")"
                         "/items/"
                         "("
                         "?P<item_id>"
                         "[^\\;\\/\\?\\:\\@\\&\\=\\+\\$\\,]+"
                         ")",
                         p2p("/collections/{collection_id}/items/{item_id}"))

        self.assertEqual("/s3bucket/"
                         "("
                         "?P<dataset_id>"
                         "[^\\;\\/\\?\\:\\@\\&\\=\\+\\$\\,]+"
                         ")"
                         "\\/?"
                         "(?P<path>.*)",
                         p2p("/s3bucket/{dataset_id}/{*path}"))

    def test_path_to_pattern_fails(self):
        p2p = TornadoFramework.path_to_pattern

        with self.assertRaises(ValueError) as cm:
            p2p('/datasets/{dataset id}')
        self.assertEqual('"{name}" in path must be a valid identifier,'
                         ' but got "dataset id"',
                         f"{cm.exception}")

        with self.assertRaises(ValueError) as cm:
            p2p('/datasets/{dataset_id')
        self.assertEqual('missing closing "}"'
                         ' in "/datasets/{dataset_id"',
                         f"{cm.exception}")

        with self.assertRaises(ValueError) as cm:
            p2p('/datasets/dataset_id}/bbox')
        self.assertEqual('missing opening "{"'
                         ' in "/datasets/dataset_id}/bbox"',
                         f"{cm.exception}")

        with self.assertRaises(ValueError) as cm:
            p2p('/datasets/{*dataset_id}/places')
        self.assertEqual('wildcard variable must be last in path,'
                         ' but path was "/datasets/{*dataset_id}/places"',
                         f"{cm.exception}")

        with self.assertRaises(ValueError) as cm:
            p2p('/datasets/{*dataset_id}/{*places}')
        self.assertEqual('only a single wildcard variable is allowed,'
                         ' but found 2 in path'
                         ' "/datasets/{*dataset_id}/{*places}"',
                         f"{cm.exception}")


# class TornadoRequestHandlerTest(unittest.TestCase):
#     def test_basic_props(self):
#         TornadoRequestHandler()


class TornadoApiRequestTest(unittest.TestCase):
    def test_basic_props(self):
        body = b'{"type": "feature", "id": 137}'
        tr = tornado.httputil.HTTPServerRequest(
            method='GET',
            host='localhost:8080',
            uri='/datasets',
            body=body
        )
        request = TornadoApiRequest(tr)
        self.assertEqual("http://localhost:8080/datasets", request.url)
        self.assertEqual(body, request.body)
        self.assertEqual({"type": "feature", "id": 137}, request.json)
        self.assertEqual([],
                         request.get_query_args('details'))

    def test_url_for_path(self):
        tr = tornado.httputil.HTTPServerRequest(
            method='GET',
            host='localhost:8080',
            uri='/datasets?details=1',
        )
        request = TornadoApiRequest(tr)
        self.assertEqual("http://localhost:8080/collections",
                         request.url_for_path('collections'))
        self.assertEqual("http://localhost:8080/collections?details=1",
                         request.url_for_path('/collections',
                                              query='details=1'))

    def test_get_query_args(self):
        tr = tornado.httputil.HTTPServerRequest(
            method='GET',
            host='localhost:8080',
            uri='/datasets?details=1',
        )
        request = TornadoApiRequest(tr)
        self.assertEqual(['1'],
                         request.get_query_args('details'))
        self.assertEqual(['1'],
                         request.get_query_args('details', type=str))
        self.assertEqual([1],
                         request.get_query_args('details', type=int))
        self.assertEqual([True],
                         request.get_query_args('details', type=bool))

    def test_get_query_args_invalid_type(self):
        tr = tornado.httputil.HTTPServerRequest(
            method='GET',
            host='localhost:8080',
            uri='/datasets?details=x',
        )
        request = TornadoApiRequest(tr)
        with self.assertRaises(ApiError.BadRequest) as cm:
            request.get_query_args('details', type=int)
        self.assertEqual("HTTP status 400:"
                         " Query parameter 'details'"
                         " must have type 'int'.",
                         f'{cm.exception}')

    def test_invalid_json(self):
        tr = tornado.httputil.HTTPServerRequest(
            method='GET',
            host='localhost:8080',
            uri='/datasets?details=x',
        )
        request = TornadoApiRequest(tr)
        with self.assertRaises(ApiError.BadRequest) as cm:
            # noinspection PyUnusedLocal
            result = request.json
        self.assertEqual("HTTP status 400:"
                         " Body does not contain valid JSON:"
                         " Expecting value: line 1 column 1 (char 0)",
                         f'{cm.exception}')


class TornadoRequestHandlerTest(unittest.TestCase):

    # noinspection PyMethodMayBeStatic
    def test_api_error_converted(self):
        application = tornado.web.Application()

        setattr(application, SERVER_CTX_ATTR_NAME, MockContext())

        # noinspection PyTypeChecker
        request = tornado.httputil.HTTPServerRequest(
            method='GET',
            host='localhost:8080',
            uri='/datasets?details=x',
            connection=MockConnection()
        )

        class TestHandler(ApiHandler):
            def get(self):
                raise ApiError(550)

            def post(self):
                raise ApiError(551)

            def put(self):
                raise ApiError(552)

            def delete(self):
                raise ApiError(553)

            def options(self):
                raise ApiError(554)

        api_route = ApiRoute('test', '/test', TestHandler)
        handler = TornadoRequestHandler(application, request,
                                        api_route=api_route)

        async def test_it():
            with pytest.raises(tornado.web.HTTPError, match=r".*550.*"):
                await handler.get()
            with pytest.raises(tornado.web.HTTPError, match=r".*551.*"):
                await handler.post()
            with pytest.raises(tornado.web.HTTPError, match=r".*552.*"):
                await handler.put()
            with pytest.raises(tornado.web.HTTPError, match=r".*553.*"):
                await handler.delete()
            with pytest.raises(tornado.web.HTTPError, match=r".*554.*"):
                await handler.options()

        import asyncio
        asyncio.run(test_it())


# Helpers

class MockIOLoop:
    def __init__(self):
        self.start_count = 0
        self.stop_count = 0
        self.call_later_count = 0
        self.run_in_executor_count = 0

    def start(self):
        self.start_count += 1

    def stop(self):
        self.stop_count += 1

    def call_later(self, *args, **kwargs):
        self.call_later_count += 1

    def run_in_executor(self, *args, **kwargs):
        self.run_in_executor_count += 1


class MockApplication:
    def __init__(self):
        self.handlers = []
        self.listen_count = 0

    def add_handlers(self, domain: str, handlers: Sequence):
        self.handlers.extend(handlers)

    def listen(self, *args, **kwargs):
        self.listen_count += 1


class MockConnection:
    def __init__(self):
        self._cb = None

    def set_close_callback(self, cb):
        self._cb = cb


class MockContext(Context):

    def __init__(self):
        self._api = Api('test')
        self._config = {}

    @property
    def apis(self) -> Tuple[Api]:
        return self._api,

    @property
    def open_api_doc(self) -> Dict[str, Any]:
        return {}

    @property
    def config(self) -> ServerConfig:
        return self.config

    def get_api_ctx(self,
                    api_name: str,
                    cls: Optional[Type[ApiContextT]] = None) \
            -> Optional[ApiContextT]:
        return self._api if api_name == 'test' else None

    def on_update(self, prev_context: Optional["Context"]):
        pass

    def on_dispose(self):
        pass

    def call_later(self, delay: Union[int, float], callback: Callable,
                   *args, **kwargs) -> object:
        pass

    def run_in_executor(self,
                        executor: Optional[concurrent.futures.Executor],
                        function: Callable[..., ReturnT], *args: Any,
                        **kwargs: Any) -> Awaitable[ReturnT]:
        pass


# xcube/__init__.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from .version import version

__version__ = version


# xcube/constants.py

# The MIT License (MIT)
# Copyright (c) 2019 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import logging
import math

PLUGIN_ENTRY_POINT_GROUP_NAME = 'xcube_plugins'
PLUGIN_MODULE_PREFIX = 'xcube_'
PLUGIN_MODULE_NAME = 'plugin'
PLUGIN_MODULE_FUNCTION_NAME = 'init_plugin'
PLUGIN_LOAD_TIME_WARN_LIMIT = 100  # milliseconds
PLUGIN_INIT_TIME__WARN_LIMIT = 100  # milliseconds

#: The extension point identifier for input processor extensions
EXTENSION_POINT_INPUT_PROCESSORS = 'xcube.core.gen.iproc'
#: The extension point identifier for dataset I/O extensions
EXTENSION_POINT_DATASET_IOS = 'xcube.core.dsio'
#: The extension point identifier for CLI command extensions
EXTENSION_POINT_CLI_COMMANDS = 'xcube.cli'
#: The extension point identifier for data stores
EXTENSION_POINT_DATA_STORES = 'xcube.core.store'
#: The extension point identifier for data openers
EXTENSION_POINT_DATA_OPENERS = 'xcube.core.store.opener'
#: The extension point identifier for data writers
EXTENSION_POINT_DATA_WRITERS = 'xcube.core.store.writer'
#: The extension point identifier for server APIs
EXTENSION_POINT_SERVER_APIS = 'xcube.server.api'
#: The extension point identifier for server frameworks such as Tornado
EXTENSION_POINT_SERVER_FRAMEWORKS = 'xcube.server.framework'

GLOBAL_GEO_EXTENT = -180., -90., 180., 90.

WGS84_ELLIPSOID_SEMI_MAJOR_AXIS = 6378137.
EARTH_EQUATORIAL_PERIMETER = 2. * math.pi * WGS84_ELLIPSOID_SEMI_MAJOR_AXIS

CRS_WKT_EPSG_4326 = """
GEOGCS["WGS 84",
    DATUM["WGS_1984",
        SPHEROID["WGS 84",6378137,298.257223563,
            AUTHORITY["EPSG","7030"]],
        AUTHORITY["EPSG","6326"]],
    PRIMEM["Greenwich",0,
        AUTHORITY["EPSG","8901"]],
    UNIT["degree",0.01745329251994328,
        AUTHORITY["EPSG","9122"]],
    AUTHORITY["EPSG","4326"]]
"""

FORMAT_NAME_ZARR = "zarr"
FORMAT_NAME_NETCDF4 = "netcdf4"
FORMAT_NAME_CSV = "csv"
FORMAT_NAME_MEM = "mem"
FORMAT_NAME_LEVELS = "levels"
FORMAT_NAME_SCRIPT = "script"
FORMAT_NAME_EXCEL = "excel"

# Note: this list must be kept in-sync with xcube.core.reproject:NAME_TO_GDAL_RESAMPLE_ALG
RESAMPLING_METHOD_NAMES = {
    # Up-sampling
    'Nearest',
    'Bilinear',
    'Cubic',
    'CubicSpline',
    'Lanczos',

    # Down-sampling
    'Average',
    'Min',
    'Max',
    'Median',
    'Mode',
    'Q1',
    'Q3',
}

LOG_LEVEL_OFF_NAME = 'OFF'
LOG_LEVEL_OFF = logging.CRITICAL + 5

LOG_LEVEL_DETAIL_NAME = 'DETAIL'
LOG_LEVEL_DETAIL = logging.DEBUG + 5

LOG_LEVEL_TRACE_NAME = 'TRACE'
LOG_LEVEL_TRACE = logging.DEBUG - 5

logging.addLevelName(LOG_LEVEL_DETAIL, LOG_LEVEL_DETAIL_NAME)
logging.addLevelName(LOG_LEVEL_TRACE, LOG_LEVEL_TRACE_NAME)

LOG_LEVELS = [
    LOG_LEVEL_OFF_NAME,
    'CRITICAL',
    'ERROR',
    'WARNING',
    'INFO',
    LOG_LEVEL_DETAIL_NAME,
    'DEBUG',
    LOG_LEVEL_TRACE_NAME
]

DEFAULT_GENERAL_LOG_LEVEL = LOG_LEVEL_OFF
DEFAULT_XCUBE_LOG_LEVEL = logging.WARNING

GENERAL_LOG_FORMAT = '[%(levelname).1s %(asctime)s %(name)s] %(message)s'
XCUBE_LOG_FORMAT = '%(levelname)s: %(message)s'

# xcube logger
LOG = logging.getLogger('xcube')
LOG.setLevel(DEFAULT_XCUBE_LOG_LEVEL)

DEFAULT_SERVER_FRAMEWORK = "tornado"
DEFAULT_SERVER_PORT = 8080
DEFAULT_SERVER_ADDRESS = "0.0.0.0"


# xcube/plugin.py

# The MIT License (MIT)
# Copyright (c) 2020 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from xcube.constants import EXTENSION_POINT_CLI_COMMANDS
from xcube.constants import EXTENSION_POINT_DATASET_IOS
from xcube.constants import EXTENSION_POINT_DATA_OPENERS
from xcube.constants import EXTENSION_POINT_DATA_STORES
from xcube.constants import EXTENSION_POINT_DATA_WRITERS
from xcube.constants import EXTENSION_POINT_INPUT_PROCESSORS
from xcube.constants import EXTENSION_POINT_SERVER_APIS
from xcube.constants import EXTENSION_POINT_SERVER_FRAMEWORKS
from xcube.constants import FORMAT_NAME_CSV
from xcube.constants import FORMAT_NAME_MEM
from xcube.constants import FORMAT_NAME_NETCDF4
from xcube.constants import FORMAT_NAME_ZARR
from xcube.util import extension


def init_plugin(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard extensions.
    """
    _register_input_processors(ext_registry)
    _register_dataset_ios(ext_registry)
    _register_cli_commands(ext_registry)
    _register_data_stores(ext_registry)
    _register_data_accessors(ext_registry)
    _register_server_apis(ext_registry)
    _register_server_frameworks(ext_registry)


def _register_input_processors(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard input processors used by "xcube gen" and gen_cube().
    """
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.gen.iproc:DefaultInputProcessor'),
        point=EXTENSION_POINT_INPUT_PROCESSORS, name='default',
        description='Single-scene NetCDF/CF inputs in xcube standard format'
    )


def _register_dataset_ios(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard dataset I/O components used by various CLI and API functions.
    """
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.dsio:ZarrDatasetIO', call=True),
        point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_ZARR,
        description='Zarr file format (http://zarr.readthedocs.io)',
        ext='zarr', modes={'r', 'w', 'a'}
    )
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.dsio:Netcdf4DatasetIO', call=True),
        point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_NETCDF4,
        description='NetCDF-4 file format',
        ext='nc', modes={'r', 'w', 'a'}
    )
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.dsio:CsvDatasetIO', call=True),
        point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_CSV,
        description='CSV file format',
        ext='csv', modes={'r', 'w'}
    )
    ext_registry.add_extension(
        loader=extension.import_component('xcube.core.dsio:MemDatasetIO', call=True),
        point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_MEM,
        description='In-memory dataset I/O',
        ext='mem', modes={'r', 'w', 'a'}
    )


_FS_STORAGE_ITEMS = (
    ('file', 'local filesystem'),
    ('s3', 'AWS S3 compatible object storage'),
    ('memory', 'in-memory filesystem')
)

_FS_DATA_ACCESSOR_ITEMS = (
    ('dataset', 'netcdf',
     'xarray.Dataset in NetCDF format'),
    ('dataset', 'zarr',
     'xarray.Dataset in Zarr format'),
    ('dataset', 'levels',
     'xarray.Dataset in leveled Zarr format'),
    ('mldataset', 'levels',
     'xcube.core.mldataset.MultiLevelDataset in leveled Zarr format'),
    ('dataset', 'geotiff',
     'xarray.Dataset in GeoTIFF or COG format'),
    ('mldataset', 'geotiff',
     'xcube.core.mldataset.MultiLevelDataset in GeoTIFF or COG format'),
    ('geodataframe', 'shapefile',
     'gpd.GeoDataFrame in ESRI Shapefile format'),
    ('geodataframe', 'geojson',
     'gpd.GeoDataFrame in GeoJSON format'),
)

_FS_DATA_OPENER_ITEMS = _FS_DATA_ACCESSOR_ITEMS
_FS_DATA_WRITER_ITEMS = _FS_DATA_ACCESSOR_ITEMS


def _register_data_stores(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard data stores.
    """
    factory = 'xcube.core.store.fs.registry:get_fs_data_store_class'

    for storage_id, storage_description in _FS_STORAGE_ITEMS:
        loader = extension.import_component(factory, call_args=[storage_id])
        ext_registry.add_extension(
            point=EXTENSION_POINT_DATA_STORES,
            loader=loader,
            name=storage_id,
            description=f'Data store that uses a {storage_description}'
        )


def _register_data_accessors(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard data accessors.
    """
    factory = 'xcube.core.store.fs.registry:get_fs_data_accessor_class'

    # noinspection PyShadowingNames
    def _add_fs_data_accessor_ext(point: str,
                                  ext_type: str,
                                  protocol: str,
                                  data_type: str,
                                  format_id: str):
        factory_args = (protocol, data_type, format_id)
        loader = extension.import_component(factory,
                                            call_args=factory_args)
        ext_registry.add_extension(
            point=point,
            loader=loader,
            name=f'{data_type}:{format_id}:{protocol}',
            description=f'Data {ext_type} for'
                        f' a {data_accessor_description}'
                        f' in {storage_description}'
        )

    for protocol, storage_description in _FS_STORAGE_ITEMS:
        for data_type, format_id, data_accessor_description \
                in _FS_DATA_OPENER_ITEMS:
            _add_fs_data_accessor_ext(EXTENSION_POINT_DATA_OPENERS,
                                      'opener',
                                      protocol,
                                      data_type,
                                      format_id)
        for data_type, format_id, data_accessor_description \
                in _FS_DATA_WRITER_ITEMS:
            _add_fs_data_accessor_ext(EXTENSION_POINT_DATA_WRITERS,
                                      'writer',
                                      protocol,
                                      data_type,
                                      format_id)


def _register_cli_commands(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard CLI commands.
    """

    cli_command_names = [
        'chunk',
        'compute',
        'benchmark',
        'dump',
        'edit',
        'extract',
        'gen',
        'gen2',
        'genpts',
        'grid',
        'level',
        'optimize',
        'patch',
        'prune',
        'rectify',
        'resample',
        'serve',
        'tile',
        'vars2dim',
        'verify',
        'versions',

        # Experimental + Hidden
        'io',
        'serve2',
    ]

    for cli_command_name in cli_command_names:
        ext_registry.add_extension(
            loader=extension.import_component(f'xcube.cli.{cli_command_name}:{cli_command_name}'),
            point=EXTENSION_POINT_CLI_COMMANDS,
            name=cli_command_name
        )


def _register_server_apis(ext_registry: extension.ExtensionRegistry):
    """
    Register xcube's standard server APIs.
    """
    server_api_names = [
        'meta',
        '_auth',
        'places',
        'styles',
        'datasets',
        'tiles',
        'timeseries',
        'ows.wmts',
        's3',
    ]
    for api_name in server_api_names:
        ext_registry.add_extension(
            loader=extension.import_component(
                f'xcube.webapi.{api_name}:api'
            ),
            point=EXTENSION_POINT_SERVER_APIS,
            name=api_name
        )


def _register_server_frameworks(ext_registry: extension.ExtensionRegistry):
    server_framework_names = [
        'tornado',
        'flask',
    ]
    for framework_name in server_framework_names:
        ext_registry.add_extension(
            loader=extension.import_component(
                    f'xcube.server.webservers.{framework_name}'
                    f':{framework_name.capitalize()}Framework',
                ),
            point=EXTENSION_POINT_SERVER_FRAMEWORKS,
            name=framework_name
        )


# xcube/version.py

# The MIT License (MIT)
# Copyright (c) 2021 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

version = '0.12.0.dev0'


# test/webapi/controllers/ogc/__init__.py

#  The MIT License (MIT)
#  Copyright (c) 2022 by the xcube development team and contributors
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.


# test/webapi/controllers/ogc/test_wmts.py

#  The MIT License (MIT)
#  Copyright (c) 2022 by the xcube development team and contributors
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.

import os
import unittest

import pyproj

from test.webapi.helpers import get_res_test_dir
from test.webapi.helpers import new_test_service_context
from xcube.core.gridmapping import GridMapping
from xcube.core.tilingscheme import TilingScheme
from xcube.webapi.context import ServiceContext
from xcube.webapi.controllers.ogc.wmts import (
    get_operations_metadata_element,
    get_service_identification_element,
    get_service_provider_element,
    get_tile_matrix_set_crs84_element,
    get_tile_matrix_set_web_mercator_element,
    get_wmts_capabilities_xml,
    get_crs84_bbox,
    WMTS_CRS84_TMS_ID,
    WMTS_WEB_MERCATOR_TMS_ID
)


class WmtsControllerTest(unittest.TestCase):

    def test_get_wmts_capabilities_xml_crs84(self):
        self.maxDiff = None
        with open(os.path.join(get_res_test_dir(),
                               'WMTSCapabilities-CRS84.xml')) as fp:
            expected_xml = fp.read()
        ctx = new_test_service_context()
        actual_xml = get_wmts_capabilities_xml(ctx,
                                               'http://bibo',
                                               WMTS_CRS84_TMS_ID)
        # Do not delete, useful for debugging
        print(80 * '=')
        print(actual_xml)
        print(80 * '=')
        self.assertEqual(expected_xml, actual_xml)

    def test_get_wmts_capabilities_xml_web_mercator(self):
        self.maxDiff = None
        with open(os.path.join(get_res_test_dir(),
                               'WMTSCapabilities-OSM.xml')) as fp:
            expected_xml = fp.read()
        ctx = new_test_service_context()
        actual_xml = get_wmts_capabilities_xml(ctx,
                                               'http://bibo',
                                               WMTS_WEB_MERCATOR_TMS_ID)
        # Do not delete, useful for debugging
        print(80 * '=')
        print(actual_xml)
        print(80 * '=')
        self.assertEqual(expected_xml, actual_xml)


class WmtsControllerXmlGenTest(unittest.TestCase):
    ctx = ServiceContext('api/v1',
                         config=dict(
                             ServiceProvider=dict(
                                 ProviderName='Bibo',
                                 ServiceContact=dict(
                                     PositionName='Boss',
                                     ContactInfo=dict(
                                         Address=dict(
                                             City='NYC'
                                         ),
                                     )
                                 ),
                             )
                         ))

    def test_get_service_provider(self):
        element = get_service_provider_element(self.ctx)
        self.assertEqual(
            '<ows:ServiceProvider>\n'
            '  <ows:ProviderName>Bibo</ows:ProviderName>\n'
            '  <ows:ProviderSite xlink:href=""/>\n'
            '  <ows:ServiceContact>\n'
            '    <ows:IndividualName/>\n'
            '    <ows:PositionName>Boss</ows:PositionName>\n'
            '    <ows:ContactInfo>\n'
            '      <ows:Phone>\n'
            '        <ows:Voice/>\n'
            '        <ows:Facsimile/>\n'
            '      </ows:Phone>\n'
            '      <ows:Address>\n'
            '        <ows:DeliveryPoint/>\n'
            '        <ows:City>NYC</ows:City>\n'
            '        <ows:AdministrativeArea/>\n'
            '        <ows:PostalCode/>\n'
            '        <ows:Country/>\n'
            '        <ows:ElectronicMailAddress/>\n'
            '      </ows:Address>\n'
            '    </ows:ContactInfo>\n'
            '  </ows:ServiceContact>\n'
            '</ows:ServiceProvider>',
            element.to_xml(indent=2)
        )

    def test_get_operations_metadata(self):
        element = get_operations_metadata_element(self.ctx,
                                                  'https://bibo',
                                                  WMTS_WEB_MERCATOR_TMS_ID)
        self.assertEqual(
            '<ows:OperationsMetadata>\n'
            '  <ows:Operation name="GetCapabilities">\n'
            '    <ows:DCP>\n'
            '      <ows:HTTP>\n'
            '        <ows:Get xlink:href="https://bibo/api/v1/wmts/kvp?">\n'
            '          <ows:Constraint name="GetEncoding">\n'
            '            <ows:AllowedValues>\n'
            '              <ows:Value>KVP</ows:Value>\n'
            '            </ows:AllowedValues>\n'
            '          </ows:Constraint>\n'
            '        </ows:Get>\n'
            '        <ows:Get xlink:href="https://bibo/api/v1/wmts/'
            '1.0.0/WorldWebMercatorQuad/WMTSCapabilities.xml">\n'
            '          <ows:Constraint name="GetEncoding">\n'
            '            <ows:AllowedValues>\n'
            '              <ows:Value>REST</ows:Value>\n'
            '            </ows:AllowedValues>\n'
            '          </ows:Constraint>\n'
            '        </ows:Get>\n'
            '      </ows:HTTP>\n'
            '    </ows:DCP>\n'
            '  </ows:Operation>\n'
            '  <ows:Operation name="GetTile">\n'
            '    <ows:DCP>\n'
            '      <ows:HTTP>\n'
            '        <ows:Get xlink:href="https://bibo/api/v1/wmts/kvp?">\n'
            '          <ows:Constraint name="GetEncoding">\n'
            '            <ows:AllowedValues>\n'
            '              <ows:Value>KVP</ows:Value>\n'
            '            </ows:AllowedValues>\n'
            '          </ows:Constraint>\n'
            '        </ows:Get>\n'
            '        <ows:Get xlink:href="https://bibo/api/v1/wmts/1.0.0/">\n'
            '          <ows:Constraint name="GetEncoding">\n'
            '            <ows:AllowedValues>\n'
            '              <ows:Value>REST</ows:Value>\n'
            '            </ows:AllowedValues>\n'
            '          </ows:Constraint>\n'
            '        </ows:Get>\n'
            '      </ows:HTTP>\n'
            '    </ows:DCP>\n'
            '  </ows:Operation>\n'
            '</ows:OperationsMetadata>',
            element.to_xml(indent=2)
        )

    def test_get_tile_matrix_set_crs84_element(self):
        tiling_scheme = TilingScheme.GEOGRAPHIC.derive(
            min_level=0,
            max_level=2
        )
        element = get_tile_matrix_set_crs84_element(tiling_scheme)
        self.assertEqual(
            '<TileMatrixSet>\n'
            '  <ows:Identifier>WorldCRS84Quad</ows:Identifier>\n'
            '  <ows:Title>CRS84 for the World</ows:Title>\n'
            '  <ows:SupportedCRS>urn:ogc:def:crs:OGC:1.3:CRS84'
            '</ows:SupportedCRS>\n'
            '  <ows:BoundingBox crs="urn:ogc:def:crs:OGC:1.3:CRS84">\n'
            '    <ows:LowerCorner>-180 -90</ows:LowerCorner>\n'
            '    <ows:UpperCorner>180 90</ows:UpperCorner>\n'
            '  </ows:BoundingBox>\n'
            '  <WellKnownScaleSet>urn:ogc:def:wkss:OGC:1.0:GoogleCRS84Quad'
            '</WellKnownScaleSet>\n'
            '  <TileMatrix>\n'
            '    <ows:Identifier>0</ows:Identifier>\n'
            '    <ScaleDenominator>279541132.01435894</ScaleDenominator>\n'
            '    <TopLeftCorner>-180 90</TopLeftCorner>\n'
            '    <TileWidth>256</TileWidth>\n'
            '    <TileHeight>256</TileHeight>\n'
            '    <MatrixWidth>2</MatrixWidth>\n'
            '    <MatrixHeight>1</MatrixHeight>\n'
            '  </TileMatrix>\n'
            '  <TileMatrix>\n'
            '    <ows:Identifier>1</ows:Identifier>\n'
            '    <ScaleDenominator>139770566.00717947</ScaleDenominator>\n'
            '    <TopLeftCorner>-180 90</TopLeftCorner>\n'
            '    <TileWidth>256</TileWidth>\n'
            '    <TileHeight>256</TileHeight>\n'
            '    <MatrixWidth>4</MatrixWidth>\n'
            '    <MatrixHeight>2</MatrixHeight>\n'
            '  </TileMatrix>\n'
            '  <TileMatrix>\n'
            '    <ows:Identifier>2</ows:Identifier>\n'
            '    <ScaleDenominator>69885283.00358973</ScaleDenominator>\n'
            '    <TopLeftCorner>-180 90</TopLeftCorner>\n'
            '    <TileWidth>256</TileWidth>\n'
            '    <TileHeight>256</TileHeight>\n'
            '    <MatrixWidth>8</MatrixWidth>\n'
            '    <MatrixHeight>4</MatrixHeight>\n'
            '  </TileMatrix>\n'
            '</TileMatrixSet>',
            element.to_xml(indent=2)
        )

    def test_get_tile_matrix_set_web_mercator_element(self):
        tiling_scheme = TilingScheme.WEB_MERCATOR.derive(
            min_level=0,
            max_level=2
        )
        element = get_tile_matrix_set_web_mercator_element(tiling_scheme)
        self.assertEqual(
            ('<TileMatrixSet>\n'
             '  <ows:Identifier>WorldWebMercatorQuad</ows:Identifier>\n'
             '  <ows:Title>Google Maps Compatible for the World</ows:Title>\n'
             '  <ows:SupportedCRS>'
             'urn:ogc:def:crs:EPSG::3857</ows:SupportedCRS>\n'
             '  <ows:BoundingBox crs="urn:ogc:def:crs:EPSG::3857">\n'
             '    <ows:LowerCorner>'
             '-20037508.3427892 -20037508.3427892</ows:LowerCorner>\n'
             '    <ows:UpperCorner>'
             '20037508.3427892 20037508.3427892</ows:UpperCorner>\n'
             '  </ows:BoundingBox>\n'
             '  '
             '<WellKnownScaleSet>'
             'urn:ogc:def:wkss:OGC:1.0:GoogleMapsCompatible'
             '</WellKnownScaleSet>\n'
             '  <TileMatrix>\n'
             '    <ows:Identifier>0</ows:Identifier>\n'
             '    <ScaleDenominator>559082264.0287178</ScaleDenominator>\n'
             '    <TopLeftCorner>'
             '-20037508.3427892 20037508.3427892</TopLeftCorner>\n'
             '    <TileWidth>256</TileWidth>\n'
             '    <TileHeight>256</TileHeight>\n'
             '    <MatrixWidth>1</MatrixWidth>\n'
             '    <MatrixHeight>1</MatrixHeight>\n'
             '  </TileMatrix>\n'
             '  <TileMatrix>\n'
             '    <ows:Identifier>1</ows:Identifier>\n'
             '    <ScaleDenominator>279541132.0143589</ScaleDenominator>\n'
             '    <TopLeftCorner>'
             '-20037508.3427892 20037508.3427892</TopLeftCorner>\n'
             '    <TileWidth>256</TileWidth>\n'
             '    <TileHeight>256</TileHeight>\n'
             '    <MatrixWidth>2</MatrixWidth>\n'
             '    <MatrixHeight>2</MatrixHeight>\n'
             '  </TileMatrix>\n'
             '  <TileMatrix>\n'
             '    <ows:Identifier>2</ows:Identifier>\n'
             '    <ScaleDenominator>139770566.00717944</ScaleDenominator>\n'
             '    <TopLeftCorner>'
             '-20037508.3427892 20037508.3427892</TopLeftCorner>\n'
             '    <TileWidth>256</TileWidth>\n'
             '    <TileHeight>256</TileHeight>\n'
             '    <MatrixWidth>4</MatrixWidth>\n'
             '    <MatrixHeight>4</MatrixHeight>\n'
             '  </TileMatrix>\n'
             '</TileMatrixSet>'),
            element.to_xml(indent=2)
        )

    def test_service_identification_element(self):
        element = get_service_identification_element()
        self.assertEqual(
            (
                '<ows:ServiceIdentification>\n'
                '  <ows:Title>xcube WMTS</ows:Title>\n'
                '  <ows:Abstract>Web Map Tile Service (WMTS)'
                ' for xcube-conformant data cubes</ows:Abstract>\n'
                '  <ows:Keywords>\n'
                '    <ows:Keyword>tile</ows:Keyword>\n'
                '    <ows:Keyword>tile matrix set</ows:Keyword>\n'
                '    <ows:Keyword>map</ows:Keyword>\n'
                '  </ows:Keywords>\n'
                '  <ows:ServiceType>OGC WMTS</ows:ServiceType>\n'
                '  <ows:ServiceTypeVersion>1.0.0</ows:ServiceTypeVersion>\n'
                '  <ows:Fees>none</ows:Fees>\n'
                '  <ows:AccessConstraints>none</ows:AccessConstraints>\n'
                '</ows:ServiceIdentification>'
            ),
            element.to_xml(indent=2)
        )


class WmtsCrs84BboxTest(unittest.TestCase):
    def test_get_crs84_bbox_ok(self):
        t = pyproj.Transformer.from_crs('EPSG:4326', 'EPSG:3035',
                                        always_xy=True)
        p0 = t.transform(10, 50)

        gm = GridMapping.regular((100, 100),
                                 p0,
                                 (1000., 1000.),
                                 pyproj.CRS.from_string('EPSG:3035'))
        bbox = get_crs84_bbox(gm)
        self.assertIsInstance(bbox, tuple)
        self.assertEqual(4, len(bbox))
        self.assertAlmostEqual(10.0, bbox[0])
        self.assertAlmostEqual(49.991462404901604, bbox[1])
        self.assertAlmostEqual(11.421266448415976, bbox[2])
        self.assertAlmostEqual(50.8990377520989, bbox[3])

    def test_get_crs84_bbox_fail(self):
        # construct impossible GridMapping
        gm = GridMapping.regular((100, 100),
                                 (100000000, -1000000),
                                 (1000., 1000.),
                                 pyproj.CRS.from_string('EPSG:3035'))
        with self.assertRaises(ValueError):
            get_crs84_bbox(gm)


# test/webapi/res/test/script.py

import numpy as np
import xarray as xr


def compute_dataset(ds, period='1W', incl_stdev=False):
    if incl_stdev:
        resample_obj = ds.resample(time=period)
        ds_mean = resample_obj.mean(dim='time')
        ds_std = resample_obj.std(dim='time').rename(name_dict={name: f"{name}_stdev" for name in ds.data_vars})
        ds_merged = xr.merge([ds_mean, ds_std])
        ds_merged.attrs.update(ds.attrs)
        return ds_merged
    else:
        return ds.resample(time=period).mean(dim='time')


def compute_variables(ds, factor_chl, factor_tsm):
    chl_tsm_sum = factor_chl * ds.conc_chl + factor_tsm * ds.conc_tsm
    chl_tsm_sum.attrs.update(dict(units='-',
                                  long_name='Weighted sum of CHL nd TSM concentrations',
                                  description='Nonsense variable, for demo purpose only'))

    chl_category = _categorize_chl(ds.conc_chl)
    chl_category.attrs.update(dict(units='-',
                                   long_name='Categorized CHL',
                                   description='0: 0<=CHL<3, 1: 3<=CHL<4, 2: CHL>4 mg/m^3'))

    return xr.Dataset(dict(chl_tsm_sum=chl_tsm_sum,
                           chl_category=chl_category))


def _categorize_chl(chl):
    return xr.where(chl >= 4., 2,
                    xr.where(chl >= 3.0, 1,
                             xr.where(chl >= 0.0, 0,
                                      np.nan)))


# test/cli/helpers.py

import unittest
from abc import ABCMeta
from typing import List, Dict, Optional

import click
import click.testing

from xcube.core.new import new_cube
from xcube.cli.main import cli
from xcube.core.dsio import rimraf

TEST_NC_FILE = "test.nc"
TEST_ZARR_DIR = "test.zarr"


class CliTest(unittest.TestCase, metaclass=ABCMeta):

    def invoke_cli(self, args: List[str]):
        self.runner = click.testing.CliRunner(mix_stderr=False)
        # noinspection PyTypeChecker
        return self.runner.invoke(cli,
                                  args,
                                  catch_exceptions=False)


class CliDataTest(CliTest, metaclass=ABCMeta):

    def outputs(self) -> List[str]:
        return []

    def time_periods(self) -> int:
        return 5

    def chunks(self) -> Optional[Dict[str, int]]:
        return None

    def setUp(self):
        self._rm_outputs()
        dataset = new_cube(variables=dict(precipitation=0.4, temperature=275.2, soil_moisture=0.5),
                           time_periods=self.time_periods())
        if self.chunks() is not None:
            dataset = dataset.chunk(self.chunks())
        dataset.to_netcdf(TEST_NC_FILE, mode="w")
        dataset.to_zarr(TEST_ZARR_DIR, mode="w")

    def tearDown(self):
        self._rm_outputs()

    def _rm_outputs(self):
        for path in [TEST_ZARR_DIR, TEST_NC_FILE] + self.outputs():
            rimraf(path)


# test/cli/test_benchmark.py

import os
import unittest

from test.cli.helpers import CliDataTest


@unittest.skip('Very slow tests, execute on demand only')
class BenchmarkCliTest(CliDataTest):

    def test_help_option(self):
        result = self.invoke_cli(['benchmark', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_simple(self):
        config_path = os.path.join(os.path.dirname(__file__), 'benchmark-configs', 'simple.yml')
        result = self.invoke_cli(['benchmark', config_path])
        msg = f'actual output:\n{result.stdout}'
        self.assertTrue('# command template: xcube dump ${input}\n' in result.stdout, msg=msg)
        self.assertTrue('\n# repetition count: 1\n' in result.stdout, msg=msg)
        self.assertTrue('\nid;input;time\n' in result.stdout, msg=msg)
        self.assertTrue('\n0;test.zarr;' in result.stdout, msg=msg)
        self.assertTrue('\n1;test.nc;' in result.stdout, msg=msg)

    def test_simple_with_repetitions(self):
        config_path = os.path.join(os.path.dirname(__file__), 'benchmark-configs', 'simple.yml')
        result = self.invoke_cli(['benchmark', '--repeats', 3, config_path])
        msg = f'actual output:\n{result.stdout}'

        self.assertTrue('# command template: xcube dump ${input}\n' in result.stdout, msg=msg)
        self.assertTrue('\n# repetition count: 3\n' in result.stdout, msg=msg)
        self.assertTrue('\nid;input;time-mean;time-median;time-stdev;time-min;time-max\n' in result.stdout, msg=msg)
        self.assertTrue('\n0;test.zarr;' in result.stdout, msg=msg)
        self.assertTrue('\n1;test.nc;' in result.stdout, msg=msg)


# test/cli/test_chunk.py

import os

import xarray as xr

from test.cli.helpers import CliDataTest, TEST_NC_FILE, TEST_ZARR_DIR


class ChunkTest(CliDataTest):
    TEST_OUTPUT = "test-chunked.zarr"

    def outputs(self):
        return [self.TEST_OUTPUT]

    def test_chunk_zarr(self):
        output_path = ChunkTest.TEST_OUTPUT
        result = self.invoke_cli(["chunk",
                                  TEST_ZARR_DIR,
                                  "-o", output_path,
                                  "--chunks", "time=1,lat=20,lon=40"])
        self.assertEqual("", result.output)
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(output_path))

        ds = xr.open_zarr(output_path)
        self.assertIn("precipitation", ds)
        precipitation = ds["precipitation"]
        self.assertTrue(hasattr(precipitation, "encoding"))
        self.assertIn("chunks", precipitation.encoding)
        self.assertEqual(precipitation.encoding["chunks"], (1, 20, 40))

    # TODO (forman): this test fails
    # netCDF4\_netCDF4.pyx:2437: in netCDF4._netCDF4.Dataset.createVariable
    # ValueError: cannot specify chunksizes for a contiguous dataset
    #
    # def test_chunk_nc(self):
    #     output_path = "test-chunked.nc"
    #     result = self.invoke_cli(["chunk",
    #                               TEST_NC_FILE,
    #                               "-o", output_path,
    #                               "--chunks", "time=1,lat=20,lon=40"])
    #     self.assertEqual("", result.output)
    #     self.assertEqual(0, result.exit_code)
    #     self.assertTrue(os.path.isdir(output_path))
    #     try:
    #         ds = xr.open_zarr(output_path)
    #         self.assertIn("precipitation", ds)
    #         precipitation = ds["precipitation"]
    #         self.assertTrue(hasattr(precipitation, "encoding"))
    #         self.assertIn("chunksizes", precipitation.encoding)
    #         self.assertEqual(precipitation.encoding["chunksizes"], (1, 20, 40))
    #     finally:
    #         os.remove(output_path)

    def test_chunk_size_syntax(self):
        result = self.invoke_cli(["chunk",
                                  TEST_NC_FILE,
                                  "-o", "test-chunked.zarr",
                                  "--chunks", "time=1,lat!gnnn,lon=40"])
        self.assertEqual("Error: Invalid value for CHUNKS:"
                         " 'time=1,lat!gnnn,lon=40'\n",
                         result.stderr)
        self.assertEqual(1, result.exit_code)

    def test_chunk_size_not_an_int(self):
        result = self.invoke_cli(["chunk",
                                  TEST_NC_FILE,
                                  "-o", "test-chunked.zarr",
                                  "--chunks", "time=1,lat=20.3,lon=40"])
        self.assertEqual("Error: Invalid value for CHUNKS,"
                         " chunk sizes must be positive integers:"
                         " time=1,lat=20.3,lon=40\n",
                         result.stderr)
        self.assertEqual(1, result.exit_code)

    def test_chunk_size_not_a_dim(self):
        result = self.invoke_cli(["chunk",
                                  TEST_NC_FILE,
                                  "-o", "test-chunked.zarr",
                                  "--chunks", "time=1,lati=20,lon=40"])
        self.assertEqual("Error: Invalid value for CHUNKS,"
                         " 'lati' is not the name of any dimension:"
                         " time=1,lati=20,lon=40\n",
                         result.stderr)
        self.assertEqual(1, result.exit_code)


# test/cli/test_common.py

import unittest

import click

from xcube.cli.common import assert_positive_int_item
from xcube.cli.common import handle_cli_exception
from xcube.cli.common import parse_cli_kwargs
from xcube.cli.common import parse_cli_sequence


class ClickUtilTest(unittest.TestCase):

    def test_parse_cli_kwargs(self):
        self.assertEqual(dict(),
                         parse_cli_kwargs("", metavar="<chunks>"))
        self.assertEqual(dict(time=1, lat=256, lon=512),
                         parse_cli_kwargs("time=1, lat=256, lon=512", metavar="<chunks>"))
        self.assertEqual(dict(chl_conc=(0, 20, 'greens'), chl_tsm=(0, 15, 'viridis')),
                         parse_cli_kwargs("chl_conc=(0,20,'greens'),chl_tsm=(0,15,'viridis')",
                                          metavar="<styles>"))

        with self.assertRaises(click.ClickException) as cm:
            parse_cli_kwargs("45 * 'A'", metavar="<chunks>")
        self.assertEqual("Invalid value for <chunks>: \"45 * 'A'\"",
                         f"{cm.exception}")

        with self.assertRaises(click.ClickException) as cm:
            parse_cli_kwargs("9==2")
        self.assertEqual("Invalid value: '9==2'",
                         f"{cm.exception}")

    def test_parse_cli_sequence(self):
        self.assertEqual((512, 1024),
                         parse_cli_sequence('512, 1024',
                                            metavar='X',
                                            item_parser=int))
        self.assertEqual((11, 11, 11),
                         parse_cli_sequence('11',
                                            metavar='X',
                                            item_parser=int,
                                            num_items=3))
        self.assertEqual((11, 12, 13),
                         parse_cli_sequence(['11', '12 ', ' 13'],
                                            metavar='X',
                                            item_parser=int))
        self.assertEqual(None,
                         parse_cli_sequence(None, allow_none=True))
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence(None,
                               metavar='X',
                               allow_none=False)
        self.assertEqual("X must be given", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11',
                               metavar='X',
                               item_parser=int,
                               num_items_min=3,
                               item_plural_name='lollies')
        self.assertEqual("X must have at least 3 lollies separated by ','", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11,12,13',
                               metavar='X',
                               item_parser=int,
                               num_items_max=2,
                               item_plural_name='lollies')
        self.assertEqual("X must have no more than 2 lollies separated by ','", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11,12,13',
                               metavar='X',
                               item_parser=int,
                               num_items=2,
                               item_plural_name='lollies')
        self.assertEqual("X must have 2 lollies separated by ','", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11,,13',
                               metavar='X',
                               item_parser=int,
                               num_items=3,
                               item_plural_name='lollies',
                               allow_empty_items=False)
        self.assertEqual("lollies in X must not be empty", f'{cm.exception}')
        with self.assertRaises(click.ClickException) as cm:
            parse_cli_sequence('11,12,-13',
                               metavar='X',
                               item_parser=int,
                               item_validator=assert_positive_int_item,
                               num_items=3,
                               item_plural_name='lollies')
        self.assertEqual("Invalid lollies in X found: all items must be positive integer numbers", f'{cm.exception}')
        with self.assertRaises(ValueError) as cm:
            parse_cli_sequence('11,12,-13',
                               metavar='X',
                               item_parser=int,
                               item_validator=assert_positive_int_item,
                               num_items=3,
                               item_plural_name='lollies',
                               error_type=ValueError)
        self.assertEqual("Invalid lollies in X found: all items must be positive integer numbers", f'{cm.exception}')

    def test_handle_cli_exception(self):
        self.assertEqual(1, handle_cli_exception(click.Abort(), traceback_mode=True))
        self.assertEqual(1, handle_cli_exception(click.ClickException("bad handle"), traceback_mode=True))
        self.assertEqual(2, handle_cli_exception(OSError("bad handle"), traceback_mode=True))
        self.assertEqual(3, handle_cli_exception(ValueError("bad handle"), traceback_mode=True))
        self.assertEqual(20, handle_cli_exception(click.Abort(), exit_code=20))
        self.assertEqual(20, handle_cli_exception(click.ClickException("bad handle"), exit_code=20))
        self.assertEqual(20, handle_cli_exception(OSError("bad handle"), exit_code=20))
        self.assertEqual(20, handle_cli_exception(ValueError("bad handle"), exit_code=20))


# test/cli/test_compute.py

import os
from typing import List

import xarray as xr

from test.cli.helpers import CliDataTest, TEST_ZARR_DIR

OUTPUT_PATH = 'out.zarr'


class ComputeCliTest(CliDataTest):

    def outputs(self) -> List[str]:
        return [OUTPUT_PATH]

    def test_help_option(self):
        result = self.invoke_cli(['compute', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_compute_with_init_and_finalize(self):
        result = self.invoke_cli(['compute',
                                  os.path.join(os.path.dirname(__file__), 'compute-scripts', 'with-init-and-finalize.py'),
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(OUTPUT_PATH))

        ds = xr.open_zarr(OUTPUT_PATH)
        self.assertEqual(['output'], list(ds.data_vars))
        self.assertEqual('mg/m^3', ds.output.attrs.get('units'))
        self.assertEqual('I has a bucket', ds.attrs.get('comment'))
        self.assertAlmostEqual(0.2 * 0.4 + 0.3 * 0.5, float(ds.output.mean()))

    def test_compute_without_init(self):
        result = self.invoke_cli(['compute',
                                  '--params', 'a=0.1,b=0.4',
                                  '--vars', 'precipitation,soil_moisture',
                                  os.path.join(os.path.dirname(__file__), 'compute-scripts', 'without-init.py'),
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(OUTPUT_PATH))

        ds = xr.open_zarr(OUTPUT_PATH)
        self.assertEqual(['output'], list(ds.data_vars))
        self.assertAlmostEqual(0.1 * 0.4 + 0.4 * 0.5, float(ds.output.mean()))


# test/cli/test_dump.py

from test.cli.helpers import CliDataTest, TEST_NC_FILE
import xarray as xr


class DumpTest(CliDataTest):

    def test_dump_ds(self):
        self.maxDiff = None
        with xr.set_options(display_width=80):
            result = self.invoke_cli(["dump", TEST_NC_FILE])

        # xarray v0.16.0 changed line truncation behaviour (see
        # https://github.com/pydata/xarray/issues/3759 ), so this test checks
        # the output against two expected values to cover both pre- and
        # post-change behaviour.
        output_template = (
            "<xarray.Dataset>\n"
            "Dimensions:        (time: 5, lat: 180, lon: 360, bnds: 2)\n"
            "Coordinates:\n"
            "  * lon            (lon) float64 -179.5 -178.5 -177.5 ... 177.5 178.5 179.5\n"
            "  * lat            (lat) float64 -89.5 -88.5 -87.5 -86.5 ... 86.5 87.5 88.5 89.5\n"
            "  * time           (time) datetime64[ns] 2010-01-01T12:00:00 ... 2010-01-05T1%s\n"
            "    lon_bnds       (lon, bnds) float64 ...\n"
            "    lat_bnds       (lat, bnds) float64 ...\n"
            "    time_bnds      (time, bnds) datetime64[ns] ...\n"
            "Dimensions without coordinates: bnds\n"
            "Data variables:\n"
            "    precipitation  (time, lat, lon) float64 ...\n"
            "    temperature    (time, lat, lon) float64 ...\n"
            "    soil_moisture  (time, lat, lon) float64 ...\n"
            "Attributes:\n"
            "    Conventions:           CF-1.7\n"
            "    title:                 Test Cube\n"
            "    time_coverage_start:   2010-01-01T00:00:00\n"
            "    time_coverage_end:     2010-01-06T00:00:00\n"
            "    geospatial_lon_min:    -180.0\n"
            "    geospatial_lon_max:    180.0\n"
            "    geospatial_lon_units:  degrees_east\n"
            "    geospatial_lat_min:    -90.0\n"
            "    geospatial_lat_max:    90.0\n"
            "    geospatial_lat_units:  degrees_north\n"
        )

        possible_outputs = [
            output_template % '2:00:00',  # for xarray v0.15.1 and below
            output_template % '...'  # for xarray v0.16.0 and above
        ]
        print(result.output)
        self.assertIn(result.output, possible_outputs)
        self.assertEqual(0, result.exit_code)


# test/cli/test_edit.py

import yaml
import zarr

from test.cli.helpers import CliDataTest
from test.core.test_edit import TEST_CUBE, TEST_CUBE_ZARR, TEST_CUBE_ZARR_EDIT, TEST_NEW_META, TEST_NEW_META_YML, \
    TEST_CUBE_ZARR_COORDS, TEST_CUBE_COORDS
from xcube.core.dsio import rimraf

TEST_CUBE_ZARR_EDIT_DEFAULT = 'test-edited.zarr'


class EditMetadaDataTest(CliDataTest):
    def _clear_outputs(self):
        rimraf(TEST_CUBE_ZARR)
        rimraf(TEST_CUBE_ZARR_EDIT)
        rimraf(TEST_NEW_META_YML)
        rimraf(TEST_CUBE_ZARR_EDIT_DEFAULT)
        rimraf(TEST_CUBE_ZARR_COORDS)

    def setUp(self):
        self._clear_outputs()
        TEST_CUBE.to_zarr(TEST_CUBE_ZARR)
        TEST_CUBE_COORDS.to_zarr(TEST_CUBE_ZARR_COORDS)
        with open(TEST_NEW_META_YML, 'w') as outfile:
            yaml.dump(TEST_NEW_META, outfile, default_flow_style=False)

    def tearDown(self):
        self._clear_outputs()

    def test_defaults(self):
        result = self.invoke_cli(['edit', TEST_CUBE_ZARR, '-M', TEST_NEW_META_YML])
        self.assertEqual(0, result.exit_code)

        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT_DEFAULT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertNotIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_name', ds2.attrs.keys())

    def test_user_output(self):
        result = self.invoke_cli(['edit', TEST_CUBE_ZARR, '-M', TEST_NEW_META_YML, '-o', TEST_CUBE_ZARR_EDIT])
        self.assertEqual(0, result.exit_code)

        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertNotIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_name', ds2.attrs.keys())

    def test_in_place(self):
        result = self.invoke_cli(['edit', '-I', TEST_CUBE_ZARR, '-M', TEST_NEW_META_YML])
        self.assertEqual(0, result.exit_code)

        ds1 = zarr.open(TEST_CUBE_ZARR)
        self.assertEqual(36, ds1.__len__())
        self.assertEqual('14-APR-2017 10:27:50.183264', ds1.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds1['conc_chl'].attrs.__getitem__('units'))
        self.assertIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_url', ds1.attrs.keys())
        self.assertEqual('Brockmann Consult GmbH with love', ds1.attrs.__getitem__('creator_name'))
        self.assertEqual('www.some_very_nice_url.com', ds1.attrs.__getitem__('creator_url'))

    def test_update_coords_only(self):
        ds1 = zarr.open(TEST_CUBE_ZARR_COORDS)
        delete_list = ['geospatial_lat_max', 'geospatial_lat_min', 'geospatial_lat_units', 'geospatial_lon_max',
                       'geospatial_lon_min', 'geospatial_lon_units', 'time_coverage_end', 'time_coverage_start']
        for attr in ds1.attrs.keys():
            if attr in delete_list:
                ds1.attrs.__delitem__(attr)
        result = self.invoke_cli(['edit', TEST_CUBE_ZARR_COORDS, '-o', TEST_CUBE_ZARR_EDIT, '-C'])
        self.assertEqual(0, result.exit_code)

        ds1 = zarr.open(TEST_CUBE_ZARR_COORDS)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        for attr in delete_list:
            self.assertNotIn(attr, ds1.attrs.keys())
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertIn('geospatial_lat_max', ds2.attrs.keys())
        self.assertIn('geospatial_lat_min', ds2.attrs.keys())
        self.assertIn('geospatial_lat_units', ds2.attrs.keys())
        self.assertIn('geospatial_lon_max', ds2.attrs.keys())
        self.assertEqual(180.0, ds2.attrs.__getitem__('geospatial_lon_max'))
        self.assertEqual(-180.0, ds2.attrs.__getitem__('geospatial_lon_min'))
        self.assertEqual('2010-01-04T00:00:00', ds2.attrs.__getitem__('time_coverage_end'))
        self.assertEqual('2010-01-01T00:00:00', ds2.attrs.__getitem__('time_coverage_start'))
        self.assertEqual('degrees_east', ds2.attrs.__getitem__('geospatial_lon_units'))


class EditTest(CliDataTest):

    def test_help_option(self):
        result = self.invoke_cli(['edit', '--help'])
        self.assertEqual(0, result.exit_code)


# test/cli/test_extract.py

import os
import os.path
import warnings
from io import StringIO

import pandas as pd

from test.cli.helpers import CliDataTest

VAR_COLS = [
    'c2rcc_flags',
    'conc_chl',
    'conc_tsm',
    'kd489',
    'quality_flags',
]

COORDS_COLS = [
    'lon',
    'lat',
    'time',
]

BOUNDS_COLS = [
    'lon_lower',
    'lon_upper',
    'lat_lower',
    'lat_upper',
    'time_lower',
    'time_upper',
]

INDEX_COLS = [
    'lon_index',
    'lat_index',
    'time_index',
]

REF_COLS = [
    'id_ref',
    'lon_ref',
    'lat_ref',
    'time_ref',
]


class ExtractCliTest(CliDataTest):

    @classmethod
    def setUpClass(cls):
        # We parse CSV-output from CLI using pandas in these tests.
        # Warnings will cause the pandas parser to fail.
        warnings.filterwarnings("ignore")

    @classmethod
    def tearDownClass(cls):
        warnings.resetwarnings()

    def test_help_option(self):
        result = self.invoke_cli(['extract', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_all_invalid_points(self):
        self._do_test_points('all-invalid-points.csv')

    def test_some_invalid_points(self):
        self._do_test_points('some-invalid-points.csv')

    def test_extract_points_with_refs(self):
        self._do_test_points('all-valid-points.csv')

    def _do_test_points(self, filename):
        self._do_test_points_with_options(filename, options=[],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--coords'],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--bounds'],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--refs'],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--indexes'],
                                          expected_length=6)
        self._do_test_points_with_options(filename, options=['--coords', '--bounds', '--refs', '--indexes'],
                                          expected_length=6)

    def _do_test_points_with_options(self,
                                     filename,
                                     options=None,
                                     expected_length=0):
        base_dir = os.path.dirname(__file__)
        args = ['extract']
        if options:
            args += options
        args += [os.path.join(base_dir, '..', '..', 'examples', 'serve', 'demo', 'cube.nc'),
                 os.path.join(base_dir, 'extract-points', filename)]
        result = self.invoke_cli(args)
        self.assertEqual(0, result.exit_code)
        self.assertNotEqual('', result.stdout)
        try:
            df = pd.read_csv(StringIO(result.stdout),
                             index_col='idx')
        except pd.errors.ParserError as e:
            print(f"Failed to parse result: {e}:")
            print(result.stdout)
            raise e
        self.assertEqual(expected_length, len(df))
        expected_columns = set(VAR_COLS)
        if '--coords' in options:
            expected_columns.update(COORDS_COLS)
        if '--bounds' in options:
            expected_columns.update(BOUNDS_COLS)
        if '--refs' in options:
            expected_columns.update(REF_COLS)
        if '--indexes' in options:
            expected_columns.update(INDEX_COLS)
        self.assertEqual(expected_columns, set(df.columns))


# test/cli/test_gen.py

from test.cli.helpers import CliTest


class GenCliTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['gen', 'time_irregular', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_help(self):
        result = self.invoke_cli(['gen', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_info(self):
        result = self.invoke_cli(['gen', '--info'])
        self.assertEqual(0, result.exit_code)

    def test_default_proc_is_valid(self):
        result = self.invoke_cli(['gen', '-P', 'default', 'input.nc'])
        self.assertEqual(0, result.exit_code)

    def test_main_with_illegal_proc(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '--proc', 'gnartz'])
        self.assertEqual("Unknown input_processor_name 'gnartz'", f'{cm.exception}')

    def test_main_with_illegal_size_option(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '-S', '120,abc', 'input.nc'])
        self.assertEqual("output_size must have the form <width>,<height>,"
                         " where both values must be positive integer numbers", f'{cm.exception}')

    def test_main_with_illegal_region_option(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '-R', '50,_2,55,21', 'input.nc'])
        self.assertEqual("output_region must have the form <lon_min>,<lat_min>,<lon_max>,<lat_max>,"
                         " where all four numbers must be floating point numbers in degrees", f'{cm.exception}')
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '-R', '50,20,55', 'input.nc'])
        self.assertEqual("output_region must have the form <lon_min>,<lat_min>,<lon_max>,<lat_max>,"
                         " where all four numbers must be floating point numbers in degrees", f'{cm.exception}')

    def test_main_with_illegal_variable_option(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '--vars', ' ', 'input.nc'])
        self.assertEqual("output_variables must be a list of existing variable names", f'{cm.exception}')

    def test_main_with_illegal_config_option(self):
        with self.assertRaises(ValueError) as cm:
            self.invoke_cli(['gen', '-c', 'nonono.yml', 'input.nc'])
        self.assertEqual("Cannot find configuration 'nonono.yml'", f'{cm.exception}')




# test/cli/test_gen2.py

import json
import os
import os.path

from test.cli.helpers import CliTest
from xcube.core.dsio import rimraf

result_file = 'result.json'
result_zarr = 'out.zarr'
result_levels = 'out.levels'


class Gen2CliTest(CliTest):

    def setUp(self) -> None:
        rimraf(result_file, result_zarr, result_levels)

    def tearDown(self) -> None:
        rimraf(result_file, result_zarr, result_levels)

    def read_result_json(self):
        self.assertTrue(os.path.exists(result_file))
        with open(result_file) as fp:
            result_json = json.load(fp)
        self.assertIsInstance(result_json, dict)
        return result_json

    def test_help(self):
        result = self.invoke_cli(['gen2', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_copy_zarr_gen(self):
        request_file = os.path.join(os.path.dirname(__file__),
                                    'gen2-requests', 'copy-zarr.yml')
        result = self.invoke_cli(['gen2',
                                  '-o', result_file,
                                  request_file])
        self.assertIsNotNone(result)
        result_json = self.read_result_json()
        self.assertEqual('ok', result_json.get('status'))
        self.assertEqual(201, result_json.get('status_code'))
        self.assertIsInstance(result_json.get('message'), str)
        self.assertIn('Cube generated successfully after ',
                      result_json.get('message'))
        result = result_json.get('result')
        self.assertIsInstance(result, dict)
        self.assertEqual(result_zarr, result.get('data_id'))
        self.assertTrue(os.path.isdir(result_zarr))

    def test_copy_zarr_info(self):
        request_file = os.path.join(os.path.dirname(__file__),
                                    'gen2-requests', 'copy-zarr.yml')
        result = self.invoke_cli(['gen2',
                                  '--info',
                                  '-o', result_file,
                                  request_file])
        self.assertIsNotNone(result)
        result_json = self.read_result_json()

        self.assertEqual('ok', result_json.get('status'))
        self.assertEqual(200, result_json.get('status_code'))
        result = result_json.get('result')
        self.assertIsInstance(result, dict)

        self.assertEqual(
            {
                'dataset_descriptor': {
                    'data_id': result_zarr,
                    'data_type': 'dataset',
                    'dims': {'lat': 1000, 'lon': 2000, 'time': 15},
                    'spatial_res': 0.0024999999999977263,
                    'time_period': '1D',
                    'time_range': ['2017-01-16', '2017-01-30'],
                    'bbox': [0.0, 50.0, 5.0, 52.5],
                    'crs': 'WGS84',
                    'data_vars': {
                        'c2rcc_flags': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'c2rcc_flags'
                        },
                        'conc_chl': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'conc_chl'
                        },
                        'conc_tsm': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'conc_tsm'
                        },
                        'kd489': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'kd489'
                        },
                        'quality_flags': {
                            'dims': ['time', 'lat', 'lon'],
                            'dtype': 'float32',
                            'name': 'quality_flags'
                        }
                    },
                },
                'size_estimation': {
                    'image_size': [2000, 1000],
                    'num_bytes': 600000000,
                    'num_requests': 75,
                    'num_tiles': [1, 1],
                    'num_variables': 5,
                    'tile_size': [2000, 1000]
                }
            },
            result)
        self.assertFalse(os.path.isdir(result_zarr))

    def test_copy_levels_gen(self):
        request_file = os.path.join(os.path.dirname(__file__),
                                    'gen2-requests', 'copy-levels.yml')
        result = self.invoke_cli(['gen2',
                                  '-o', result_file,
                                  request_file])
        print(result.output)
        self.assertIsNotNone(result)
        result_json = self.read_result_json()
        self.assertEqual('ok', result_json.get('status'))
        self.assertEqual(201, result_json.get('status_code'))
        self.assertEqual({'data_id': 'out.levels'}, result_json.get('result'))
        self.assertTrue(os.path.isdir(result_levels))

    def test_internal_error(self):
        request_file = os.path.join(os.path.dirname(__file__),
                                    'gen2-requests', 'internal-error.yml')
        result = self.invoke_cli(['gen2',
                                  '-o', result_file,
                                  request_file])
        print(result.output)
        self.assertIsNotNone(result)
        result_json = self.read_result_json()
        self.assertEqual('error', result_json.get('status'))
        self.assertEqual(500, result_json.get('status_code'))
        self.assertEqual('inverse_fine_structure_constant must be 137'
                         ' or running in wrong universe',
                         result_json.get('message'))
        self.assertNotIn('result', result_json)


# test/cli/test_genpts.py

import os.path
from typing import List

import fiona
import pandas as pd

from test.cli.helpers import CliDataTest
from test.cli.helpers import TEST_ZARR_DIR

TEST_CSV = os.path.join(os.path.dirname(__file__), 'out.csv')
TEST_GEOJSON = os.path.join(os.path.dirname(__file__), 'out.geojson')


class GenptsCliTest(CliDataTest):

    def outputs(self) -> List[str]:
        return [TEST_CSV, TEST_GEOJSON]

    def test_help_option(self):
        result = self.invoke_cli(['extract', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_csv(self):
        result = self.invoke_cli(['genpts', TEST_ZARR_DIR, '--output', TEST_CSV])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('', result.stdout)
        df = pd.read_csv(TEST_CSV)
        self.assertEqual(100, len(df))

    def test_geojson(self):
        result = self.invoke_cli(['genpts', TEST_ZARR_DIR, '--output', TEST_GEOJSON])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('', result.stdout)
        with fiona.open(TEST_GEOJSON) as collection:
            features = [feature for feature in collection]
        self.assertEqual(100, len(features))


# test/cli/test_grid.py

import unittest
from fractions import Fraction
from typing import List

import click
import click.testing

from xcube.cli.grid import grid as cli
from xcube.cli.grid import factor_out_two, get_adjusted_box, get_levels, meters_to_degrees, find_close_resolutions


class GridToolsTest(unittest.TestCase):

    def test_factor_out_two(self):
        with self.assertRaises(ValueError):
            factor_out_two(-1)
        self.assertEqual((0, 0), factor_out_two(0))
        self.assertEqual((1, 0), factor_out_two(1))
        self.assertEqual((1, 1), factor_out_two(2))
        self.assertEqual((3, 0), factor_out_two(3))
        self.assertEqual((1, 2), factor_out_two(4))
        self.assertEqual((83743, 0), factor_out_two(83743))
        self.assertEqual((2617, 5), factor_out_two(83744))
        self.assertEqual(83744, 2617 * 2 ** 5)

    def test_get_adjusted_box(self):
        self.assertEqual((0.0, 49.21875, 5.625, 53.4375),
                         get_adjusted_box(0.0, 50.0, 5.0, 52.5, 540 / 384))

    def test_get_levels(self):
        results = get_levels(3, 180, level_min=4)
        self.assertEqual(6, len(results))
        self.assertEqual(['L', 0, 1, 2, 3, 4],
                         [row[0] for row in results])
        self.assertEqual(['H', 3, 6, 12, 24, 48],
                         [row[1] for row in results])
        self.assertEqual(['R', Fraction(60, 1), Fraction(30, 1), Fraction(15, 1), Fraction(15, 2), Fraction(15, 4)],
                         [row[2] for row in results])
        self.assertEqual(['R (deg)', 60.0, 30.0, 15.0, 7.5, 3.75],
                         [row[3] for row in results])
        self.assertEqual(['R (m)', 6679169.45, 3339584.72, 1669792.36, 834896.18, 417448.09],
                         [row[4] for row in results])

    def test_find_close_resolutions(self):
        target_res = meters_to_degrees(300)
        results = find_close_resolutions(target_res, target_res * 1.0 / 100, 180)
        self.assertEqual(43, len(results))
        self.assertEqual(['R_D (%)', 0.012, -0.036, 0.059, -0.084],
                         [row[0] for row in results[0:5]])
        self.assertEqual(['R_NOM', 45, 5, 45, 45],
                         [row[1] for row in results[0:5]])
        self.assertEqual(['R_DEN', 16696, 1856, 16688, 16712],
                         [row[2] for row in results[0:5]])
        self.assertEqual(['R (deg)',
                          0.002695256348826066,
                          0.0026939655172413795,
                          0.002696548418024928,
                          0.0026926759214935376],
                         [row[3] for row in results[0:5]])
        self.assertEqual(['R (m)', 300.03, 299.89, 300.18, 299.75],
                         [row[4] for row in results[0:5]])
        self.assertEqual(['H', 66784, 66816, 66752, 66848],
                         [row[5] for row in results[0:5]])
        self.assertEqual(['H0', 2087, 261, 1043, 2089],
                         [row[6] for row in results[0:5]])


class GridCliTest(unittest.TestCase):

    @classmethod
    def invoke_cli(cls, args: List[str]):
        runner = click.testing.CliRunner()
        return runner.invoke(cli, args)

    def test_levels(self):
        result = self.invoke_cli(["levels", "-R", "1/384"])
        self.assertEqual(("""
L	H	R	R (deg)	R (m)
0	135	4/3	1.3333333333333333	148425.99
1	270	2/3	0.6666666666666666	74212.99
2	540	1/3	0.3333333333333333	37106.5
3	1080	1/6	0.16666666666666666	18553.25
4	2160	1/12	0.08333333333333333	9276.62
5	4320	1/24	0.041666666666666664	4638.31
6	8640	1/48	0.020833333333333332	2319.16
7	17280	1/96	0.010416666666666666	1159.58
8	34560	1/192	0.005208333333333333	579.79
9	69120	1/384	0.0026041666666666665	289.89
"""),
                         result.output)
        self.assertEqual(0, result.exit_code)

    def test_res(self):
        result = self.invoke_cli(["res", "300m", "-D", "1%", "--sort_by", "+H0", "-N", "6"])
        print(result.output)
        self.assertEqual(("""
R_D (%)	R_NOM	R_DEN	R (deg)	R (m)	H	H0	L
0.348	9	3328	0.002704326923076923	301.04	66560	65	10
-0.418	45	16768	0.0026836832061068704	298.75	67072	131	9
0.736	45	16576	0.00271476833976834	302.21	66304	259	8
-0.036	5	1856	0.0026939655172413795	299.89	66816	261	8
-0.797	45	16832	0.0026734790874524714	297.61	67328	263	8
37 more...
"""),
                         result.output)
        self.assertEqual(0, result.exit_code)

    def test_abox(self):
        result = self.invoke_cli(['abox', "0,50,5,52.5", "-R", "1/384"])
        self.assertEqual(("""
Orig. box coord. = 0.0,50.0,5.0,52.5
Adj. box coord.  = 0.0,49.33333333333333,5.333333333333333,53.33333333333333
Orig. box WKT    = POLYGON ((0.0 50.0, 5.0 50.0, 5.0 52.5, 0.0 52.5, 0.0 50.0))
Adj. box WKT     = POLYGON ((0.0 49.33333333333333, 5.333333333333333 49.33333333333333, 5.333333333333333 53.33333333333333, 0.0 53.33333333333333, 0.0 49.33333333333333))
Combined WKT     = MULTIPOLYGON (((0.0 50.0, 5.0 50.0, 5.0 52.5, 0.0 52.5, 0.0 50.0)), ((0.0 49.33333333333333, 5.333333333333333 49.33333333333333, 5.333333333333333 53.33333333333333, 0.0 53.33333333333333, 0.0 49.33333333333333)))
Box grid size    = 2048 x 1536 cells
Graticule dist.  = 4/3 degrees
Tile size        = 1 cells
Granularity      = 135 cells
Level            = 9
Res. at level 0  = 4/3 degrees
Resolution       = 1/384 degrees
                 = 289.89 meters
"""),
                         result.output)
        self.assertEqual(0, result.exit_code)


# test/cli/test_io.py

import json
import os.path

import yaml

from test.cli.helpers import CliTest
from xcube.core.dsio import rimraf


class IOTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['io', '--help'])
        self.assertEqual(0, result.exit_code)


class IOStoreTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['io', 'store', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'store', 'list', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'store', 'info', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_list(self):
        result = self.invoke_cli(['io', 'store', 'list'])
        self.assertEqual(0, result.exit_code)

    def test_info(self):
        result = self.invoke_cli(['io', 'store', 'info', 'memory',
                                  '-POWj'])
        self.assertEqual(0, result.exit_code)
        result = self.invoke_cli(['io', 'store', 'info', 'file',
                                  '-POW', 'root=.'])
        self.assertEqual(0, result.exit_code)

        # param "root" missing
        result = self.invoke_cli(['io', 'store', 'info',
                                  'file', '-POW', 'root=42'])
        self.assertEqual(1, result.exit_code)
        self.assertEqual("Error: Failed to instantiate data store 'file'"
                         " with parameters ('root=42',):"
                         " Invalid parameterization detected:"
                         " 42 is not of type 'string'\n",
                         result.stderr)

    def test_data(self):
        demo_dir = os.path.join(os.path.dirname(__file__),
                                '..', '..',
                                'examples', 'serve', 'demo')

        result = self.invoke_cli(['io', 'store', 'info',
                                  'file', '-D', f'root={demo_dir}'])
        self.assertEqual(0, result.exit_code)
        self.assertEqual(('\n'
                          'Data store description:\n'
                          '  Data store that uses a local filesystem\n'
                          '\n'
                          'Data resources:\n'
                          '             cube-1-250-250.levels  <no title>\n'
                          '               cube-1-250-250.zarr  <no title>\n'
                          '               cube-5-100-200.zarr  <no title>\n'
                          '                           cube.nc  <no title>\n'
                          '                    sample-cog.tif  <no title>\n'
                          '                sample-geotiff.tif  <no title>\n'
                          '6 data resources found.\n'),
                         result.stdout)
        demo_dir = os.path.join(os.path.dirname(__file__),
                                '..', '..',
                                'examples', 'serve', 'demo')
        result = self.invoke_cli(['io', 'store', 'data',
                                  'file', 'cube-1-250-250.zarr',
                                  f'root={demo_dir}'])
        self.assertEqual(0, result.exit_code)
        self.assertIn('cube-1-250-250.zarr', result.stdout)


class IOOpenerTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['io', 'opener', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'opener', 'list', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'opener', 'info', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_list(self):
        result = self.invoke_cli(['io', 'opener', 'list'])
        self.assertEqual(0, result.exit_code)

    def test_info(self):
        result = self.invoke_cli(['io', 'opener', 'info', 'dataset:zarr:file'])
        self.assertEqual(0, result.exit_code)


class IOWriterTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['io', 'writer', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'writer', 'list', '--help'])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(['io', 'writer', 'info', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_list(self):
        result = self.invoke_cli(['io', 'writer', 'list'])
        self.assertEqual(0, result.exit_code)

    def test_info(self):
        result = self.invoke_cli(['io', 'writer', 'info', 'dataset:zarr:file'])
        self.assertEqual(0, result.exit_code)


class IODumpTest(CliTest):
    STORE_CONF = {
        "this_dir": {
            "store_id": "file",
            "store_params": {
                "root": os.path.join(os.path.dirname(__file__),
                                     "..", "..",
                                     "examples", "serve", "demo")
            }
        }
    }

    @classmethod
    def setUpClass(cls) -> None:
        with open('store-conf.yml', 'w') as fp:
            yaml.dump(cls.STORE_CONF, fp)
        with open('store-conf.json', 'w') as fp:
            json.dump(cls.STORE_CONF, fp)

    @classmethod
    def tearDownClass(cls) -> None:
        rimraf('store-conf.json')
        rimraf('store-conf.yml')

    def tearDown(self) -> None:
        rimraf('store-dump.json')
        rimraf('store-dump.yml')
        rimraf('store-dump.csv')
        rimraf('out.json')
        rimraf('out.jml')
        rimraf('out.txt')

    def test_help_option(self):
        result = self.invoke_cli(['io', 'dump',
                                  '--help'])
        self.assertEqual(0, result.exit_code)

    def test_yaml_config(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.json'))

    def test_yaml_config_short_format(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml',
                                  '--short', '--json'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.json'))
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml',
                                  '--short', '--yaml'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.yml'))
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml',
                                  '--short', '--csv'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.csv'))

    def test_json_config_short(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.yml',
                                  '--short'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.json'))

    def test_json_config_output(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.json',
                                  '-o', 'out.json'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('out.json'))
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.json',
                                  '-o', 'out.yml'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('out.yml'))
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.json',
                                  '-o', 'out.txt'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('out.txt'))

    def test_json_config_type(self):
        result = self.invoke_cli(['io', 'dump',
                                  '-c', 'store-conf.json',
                                  '-t', 'dataset'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.exists('store-dump.json'))


# test/cli/test_level.py

import os.path
from typing import List, Tuple, Optional, Dict

from test.cli.helpers import CliDataTest
from test.cli.helpers import CliTest
from test.cli.helpers import TEST_NC_FILE, TEST_ZARR_DIR
from xcube.core.mldataset import MultiLevelDataset
from xcube.core.store import new_fs_data_store


class LevelTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['level', '--help'])
        self.assertEqual(0, result.exit_code)


class LevelDataTest(CliDataTest):
    TEST_OUTPUT = "test.levels"

    def outputs(self) -> List[str]:
        return [LevelDataTest.TEST_OUTPUT, 'my.levels']

    def chunks(self) -> Optional[Dict[str, int]]:
        return dict(time=1, lat=90, lon=180)

    def test_all_defaults(self):
        result = self.invoke_cli(['level', '-v', TEST_ZARR_DIR])
        self._assert_result_ok(
            result, [((1, 1, 1, 1, 1), (90, 90), (180, 180)),
                     ((1, 1, 1, 1, 1), (90,), (180,))],
            LevelDataTest.TEST_OUTPUT,
            'Multi-level dataset written to /test.levels after .*\n'
        )

    def test_with_output(self):
        result = self.invoke_cli(['level', '-v', TEST_ZARR_DIR,
                                  '--output', 'my.levels'])
        self._assert_result_ok(
            result, [((1, 1, 1, 1, 1), (90, 90), (180, 180)),
                     ((1, 1, 1, 1, 1), (90,), (180,))], 'my.levels',
            'Multi-level dataset written to my.levels after .*\n'
        )

    def test_with_tile_size_and_num_levels(self):
        result = self.invoke_cli(['level', '-v', TEST_ZARR_DIR,
                                  '-t', '90,45', '-n', '4'])
        self._assert_result_ok(
            result, [((1, 1, 1, 1, 1), (45, 45, 45, 45), (90, 90, 90, 90)),
                     ((1, 1, 1, 1, 1), (45, 45), (90, 90)),
                     ((1, 1, 1, 1, 1), (45,), (90,))],
            LevelDataTest.TEST_OUTPUT,
            'Multi-level dataset written to /test.levels after .*\n'
        )

    def _assert_result_ok(self,
                          result,
                          level_chunks: List[Tuple],
                          output_path: str,
                          message_regex: str):
        self.assertEqual(0, result.exit_code)
        self.assertRegex(result.stderr, message_regex)
        self.assertTrue(os.path.isdir(output_path))
        data_store = new_fs_data_store("file")
        ml_dataset = data_store.open_data(output_path)
        self.assertIsInstance(ml_dataset, MultiLevelDataset)
        self.assertEqual(len(level_chunks), ml_dataset.num_levels)
        for level in range(ml_dataset.num_levels):
            level_dataset = ml_dataset.get_dataset(level)
            self.assertEqual({'precipitation',
                              'soil_moisture',
                              'temperature'},
                             set(level_dataset.data_vars.keys()))
            for var_name, var in level_dataset.data_vars.items():
                var_chunks = level_chunks[level]
                self.assertEqual(var_chunks,
                                 var.chunks,
                                 f'{var_name} at level {level}')

    def _assert_result_not_ok(self, result, message_regex: str):
        self.assertEqual(1, result.exit_code)
        self.assertRegex(result.stderr, message_regex)

    def test_level_with_nc(self):
        result = self.invoke_cli(["level",
                                  "-t", "45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr(self):
        result = self.invoke_cli(["level",
                                  "-t", "45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_ZARR_DIR,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr_agg_method(self):
        result = self.invoke_cli(["level",
                                  "-t", "45",
                                  "-A", "auto",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_ZARR_DIR,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr_agg_methods(self):
        result = self.invoke_cli([
            "level",
            "-t", "45",
            "-A", "precipitation=mean,temperature=max,soil_*=median",
            "-o", LevelDataTest.TEST_OUTPUT,
            TEST_ZARR_DIR,
        ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr_link(self):
        result = self.invoke_cli(["level",
                                  "--link",
                                  "-t", "45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_ZARR_DIR,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.link', '1.zarr', '2.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_level_with_zarr_num_levels_max(self):
        result = self.invoke_cli(["level",
                                  "-t", "45",
                                  "-n", "2",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_ZARR_DIR,
                                  ])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(LevelDataTest.TEST_OUTPUT))
        self.assertEqual({'0.zarr', '1.zarr'},
                         set(os.listdir(LevelDataTest.TEST_OUTPUT)))

    def test_invalid_inputs(self):
        result = self.invoke_cli(["level",
                                  "-t", "a45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE,
                                  ])
        self._assert_result_not_ok(
            result,
            "Error\\: Invalid tile sizes in TILE_SIZE found: "
            "invalid literal for int\\(\\) with base 10\\: 'a45'\n"
        )

        result = self.invoke_cli(["level",
                                  "-t", "-3",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE,
                                  ])
        self._assert_result_not_ok(
            result,
            "Error\\: Invalid tile sizes in TILE_SIZE found\\: "
            "all items must be positive integer numbers\n"
        )

        result = self.invoke_cli(["level",
                                  "-t", "45,45,45",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE])
        self._assert_result_not_ok(
            result,
            "Error\\: TILE_SIZE must have 2 tile sizes separated by ','\n"
        )

        result = self.invoke_cli(["level",
                                  "-n", "0",
                                  "-o", LevelDataTest.TEST_OUTPUT,
                                  TEST_NC_FILE])
        self._assert_result_not_ok(
            result,
            "NUM_LEVELS_MAX must be a positive integer\n"
        )

    def test_with_existing_output(self):
        # Product output
        self.invoke_cli(['level', TEST_ZARR_DIR,
                         '--output', 'my.levels'])
        # Product output once mor
        result = self.invoke_cli(['level', TEST_ZARR_DIR,
                                  '--output', 'my.levels'])
        self._assert_result_not_ok(
            result,
            'Error: output \'my\\.levels\' already exists\n'
        )


# test/cli/test_main.py

import warnings

import pytest
from click import command

import xcube.cli.main
from test.cli.helpers import CliTest


class ClickMainTest(CliTest):
    USER_WARNING_MESSAGE = 'This is a user warning.'
    DEPRECATION_WARNING_MESSAGE = 'This is a deprecation warning.'
    RUNTIME_WARNING_MESSAGE = 'This is a runtime warning.'

    def setUp(self):
        @command(name='test_warnings')
        def test_warnings():
            warnings.warn(self.USER_WARNING_MESSAGE)
            warnings.warn(self.DEPRECATION_WARNING_MESSAGE, category=DeprecationWarning)
            warnings.warn(self.RUNTIME_WARNING_MESSAGE, category=RuntimeWarning)

        xcube.cli.main.cli.add_command(test_warnings)

    def test_warnings_not_issued_by_default(self):
        with pytest.warns(None) as record:
            self.invoke_cli(['test_warnings'])
        self.assertEqual(['This is a user warning.'],
                         list(map(lambda r: str(r.message), record.list)))

    def test_warnings_issued_when_enabled(self):
        with pytest.warns(None) as record:
            self.invoke_cli(['--warnings', 'test_warnings'])
        self.assertEqual(['This is a user warning.',
                          'This is a deprecation warning.',
                          'This is a runtime warning.'],
                         list(map(lambda r: str(r.message), record.list)))


# test/cli/test_optimize.py

from test.cli.helpers import CliDataTest
from test.core.test_optimize import TEST_CUBE, INPUT_CUBE_PATH, INPUT_CUBE_FILE_SET, list_file_set
from xcube.core.dsio import rimraf

OUTPUT_CUBE_OPTIMIZED_DEFAULT_PATH = 'test-optimized.zarr'
OUTPUT_CUBE_OPTIMIZED_USER_PATH = 'fast-test.zarr'


class OptimizeDataTest(CliDataTest):
    def _clear_outputs(self):
        rimraf(INPUT_CUBE_PATH)
        rimraf(OUTPUT_CUBE_OPTIMIZED_DEFAULT_PATH)
        rimraf(OUTPUT_CUBE_OPTIMIZED_USER_PATH)

    def setUp(self):
        self._clear_outputs()
        TEST_CUBE.to_zarr(INPUT_CUBE_PATH)

    def tearDown(self):
        self._clear_outputs()

    def test_defaults(self):
        result = self.invoke_cli(['optimize', INPUT_CUBE_PATH])
        self.assertEqual(0, result.exit_code)

        expected_files = set(INPUT_CUBE_FILE_SET)
        expected_files.add('.zmetadata')
        self.assertEqual(expected_files, list_file_set(OUTPUT_CUBE_OPTIMIZED_DEFAULT_PATH))

    def test_user_output(self):
        result = self.invoke_cli(['optimize', '-o', OUTPUT_CUBE_OPTIMIZED_USER_PATH, '-C', INPUT_CUBE_PATH])
        self.assertEqual(0, result.exit_code)

        expected_files = set(INPUT_CUBE_FILE_SET)
        expected_files.add('.zmetadata')
        expected_files.remove('time/1')
        expected_files.remove('time/2')
        expected_files.remove('time_bnds/1.0')
        expected_files.remove('time_bnds/2.0')
        self.assertEqual(expected_files, list_file_set(OUTPUT_CUBE_OPTIMIZED_USER_PATH))

    def test_in_place(self):
        result = self.invoke_cli(['optimize', '-IC', INPUT_CUBE_PATH])
        self.assertEqual(0, result.exit_code)

        expected_files = set(INPUT_CUBE_FILE_SET)
        expected_files.add('.zmetadata')
        expected_files.remove('time/1')
        expected_files.remove('time/2')
        expected_files.remove('time_bnds/1.0')
        expected_files.remove('time_bnds/2.0')
        self.assertEqual(expected_files, list_file_set(INPUT_CUBE_PATH))


class OptimizeTest(CliDataTest):

    def test_help_option(self):
        result = self.invoke_cli(['optimize', '--help'])
        self.assertEqual(0, result.exit_code)


# test/cli/test_patch.py

import unittest

import click
import xarray as xr
import yaml

from test.cli.helpers import CliTest
from xcube.cli.patch import parse_metadata
from xcube.core.dsio import rimraf
from xcube.core.new import new_cube

TEST_METADATA_FILE = 'patch.yaml'
TEST_OPTIONS_FILE = 'options.yaml'


class PatchTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['patch', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_missing_cube_arg(self):
        result = self.invoke_cli(['patch'])
        self.assertEqual(2, result.exit_code)


class PatchZarrCubeTest(CliTest):
    CUBE_PATH = 'cube.zarr'
    METADATA_PATH = 'metadata.yaml'
    OPTIONS_PATH = 'options.yaml'

    def test_ok(self):
        self.write_cube()
        self.write_metadata({
            "zarr_consolidated_format": 1,
            "metadata": {
                ".zattrs": {
                    "title": "Humpty-Dumpty",
                    "TileSize": "__delete__",   # SNAP adds this, remove!
                },
                "conc_chl/.zattrs": {
                    "title": "CHL"
                },
                "conc_tsm/.zattrs": {
                    "title": "TSM"
                }
            }
        })
        self.write_options({})
        result = self.invoke_cli([
            'patch', self.CUBE_PATH,
            '--metadata', self.METADATA_PATH,
            '--options', self.OPTIONS_PATH,
        ])
        self.assertEqual(0, result.exit_code)
        cube = xr.open_zarr(self.CUBE_PATH)
        self.assertEqual(
            {'Conventions': 'CF-1.7',
             'geospatial_lat_max': 90.0,
             'geospatial_lat_min': -90.0,
             'geospatial_lat_units': 'degrees_north',
             'geospatial_lon_max': 180.0,
             'geospatial_lon_min': -180.0,
             'geospatial_lon_units': 'degrees_east',
             'time_coverage_end': '2010-01-02T00:00:00',
             'time_coverage_start': '2010-01-01T00:00:00',
             'title': 'Humpty-Dumpty'},
            cube.attrs
        )
        self.assertEqual(
            {'long_name': 'Chlorophyll concentration',
             'title': 'CHL',
             'units': 'mg m^-3'},
            cube.conc_chl.attrs
        )
        self.assertEqual(
            {'long_name': 'Total suspended matter dry weight concentration',
             'title': 'TSM',
             'units': 'g m^-3'},
            cube.conc_tsm.attrs
        )

    def test_missing_metadata_option(self):
        result = self.invoke_cli(['patch', self.CUBE_PATH])
        self.assertEqual(1, result.exit_code)

    @classmethod
    def write_options(cls, options):
        if options is not None:
            with open(cls.OPTIONS_PATH, 'w') as fp:
                yaml.dump(options, fp, indent=2)

    @classmethod
    def write_metadata(cls, metadata):
        if metadata is not None:
            with open(cls.METADATA_PATH, 'w') as fp:
                yaml.dump(metadata, fp, indent=2)

    @classmethod
    def write_cube(cls):
        cube = new_cube(width=36,
                        height=18,
                        x_res=10,
                        time_periods=1,
                        drop_bounds=True,
                        variables=dict(
                            conc_chl=0.8,
                            conc_tsm=4.2,
                        ))
        cube.conc_chl.attrs.update({
            "long_name": "Chlorophyll concentration",
            "units": "mg m^-3"
        })
        cube.conc_tsm.attrs.update({
            "long_name": "Total suspended matter dry weight concentration",
            "units": "g m^-3",
        })
        cube.attrs.update({
            "Conventions": "CF-1.7",
            "coordinates": "time lat lon",
            "TileSize": "1024:1024"  # SNAP adds this to NetCDFs
        })
        cube.to_zarr(cls.CUBE_PATH)

    def tearDown(self) -> None:
        rimraf(self.CUBE_PATH,
               self.METADATA_PATH,
               self.OPTIONS_PATH)


class MetadataTest(unittest.TestCase):
    def test_parse_metadata_ok(self):
        metadata = parse_metadata(
            {
                "zarr_consolidated_format": 1,
                "metadata": {
                    ".zattrs": {
                        "title": "Humpty-Dumpty"
                    }
                }
            }
        )
        self.assertEqual(
            {
                ".zattrs": {
                    "title": "Humpty-Dumpty"
                }
            },
            metadata
        )

    def test_parse_metadata_fails(self):
        with self.assertRaises(click.ClickException) as cm:
            parse_metadata([])
        self.assertEqual('Invalid consolidated metadata format',
                         f'{cm.exception}')

        with self.assertRaises(click.ClickException) as cm:
            parse_metadata({
                "zarr_consolidated_format": 3,
                "metadata": {}
            })
        self.assertEqual('Unsupported consolidated metadata version',
                         f'{cm.exception}')

        with self.assertRaises(click.ClickException) as cm:
            parse_metadata({
                "zarr_consolidated_format": 1,
                "metadata": []
            })
        self.assertEqual('Invalid metadata format',
                         f'{cm.exception}')

        with self.assertRaises(click.ClickException) as cm:
            parse_metadata({
                "zarr_consolidated_format": 1,
                "metadata": {
                    ".zattrs": []
                }
            })
        self.assertEqual('Invalid metadata format:'
                         ' entry ".zattrs" is not an object',
                         f'{cm.exception}')

        with self.assertRaises(click.ClickException) as cm:
            parse_metadata({
                "zarr_consolidated_format": 1,
                "metadata": {
                }
            })
        self.assertEqual('No metadata provided',
                         f'{cm.exception}')

    def test_parse_metadata_warn(self):
        metadata = parse_metadata(
            {
                "zarr_consolidated_format": 1,
                "metadata": {
                    ".zattrs": {
                        "title": "Humpty-Dumpty"
                    },
                    "i_am_invalid_attrs": {},
                    "i/am/invalid/.zattrs": {}
                }
            }
        )
        self.assertEqual(
            {
                ".zattrs": {
                    "title": "Humpty-Dumpty"
                }
            },
            metadata
        )


# test/cli/test_prune.py

import numpy as np
import os.path
import sys

import xarray as xr

from test.cli.helpers import CliTest
from xcube.cli.prune import _delete_block_file
from xcube.core.dsio import rimraf
from xcube.core.dsio import write_cube
from xcube.core.new import new_cube
from xcube.core.verify import assert_cube


class PruneTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['prune', '--help'])
        self.assertEqual(0, result.exit_code)


class PruneDataTest(CliTest):
    TEST_CUBE = "test.zarr"

    def setUp(self) -> None:
        rimraf(self.TEST_CUBE)
        cube = new_cube(time_periods=3,
                        variables=dict(precipitation=np.nan,
                                       temperature=np.nan)) \
            .chunk(dict(time=1, lat=90, lon=90))
        fv_encoding = dict(
            _FillValue=None
        )
        encoding = dict(
            precipitation=fv_encoding,
            temperature=fv_encoding
        )
        cube.to_zarr(self.TEST_CUBE, encoding=encoding)

    def tearDown(self) -> None:
        rimraf(self.TEST_CUBE)

    def test_dry_run(self):
        result = self.invoke_cli(['prune', self.TEST_CUBE, "-vv", "--dry-run"])
        self.assertEqual(0, result.exit_code)
        self.assertEqual(
            (
                "Opening dataset from 'test.zarr'...\n"
                "Identifying empty chunks...\n"
                "Found empty chunks in variable 'precipitation', "
                "deleting block files...\n"
                "Deleted 24 block file(s) for variable 'precipitation'.\n"
                "Found empty chunks in variable 'temperature', "
                "deleting block files...\n"
                "Deleted 24 block file(s) for variable 'temperature'.\n"
                "Done, 48 block file(s) deleted total.\n"
            ),
            result.stderr)
        expected_file_names = sorted([
            '.zarray',
            '.zattrs',
            '0.0.0', '0.0.1', '0.0.2', '0.0.3',
            '0.1.0', '0.1.1', '0.1.2', '0.1.3',
            '1.0.0', '1.0.1', '1.0.2', '1.0.3',
            '1.1.0', '1.1.1', '1.1.2', '1.1.3',
            '2.0.0', '2.0.1', '2.0.2', '2.0.3',
            '2.1.0', '2.1.1', '2.1.2', '2.1.3'
        ])
        self.assertEqual(expected_file_names,
                         sorted(os.listdir('test.zarr/precipitation')))
        self.assertEqual(expected_file_names,
                         sorted(os.listdir('test.zarr/temperature')))
        ds = xr.open_zarr('test.zarr')
        assert_cube(ds)
        self.assertIn('precipitation', ds)
        self.assertEqual((3, 180, 360), ds.precipitation.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.precipitation.dims)
        self.assertIn('temperature', ds)
        self.assertEqual((3, 180, 360), ds.temperature.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.temperature.dims)

    def test_no_dry_run(self):
        result = self.invoke_cli(['prune', self.TEST_CUBE, "-vv"])
        self.assertEqual(0, result.exit_code)
        self.assertEqual(
            (
                "Opening dataset from 'test.zarr'...\n"
                "Identifying empty chunks...\n"
                "Found empty chunks in variable 'precipitation', "
                "deleting block files...\n"
                "Deleted 24 block file(s) for variable 'precipitation'.\n"
                "Found empty chunks in variable 'temperature', "
                "deleting block files...\n"
                "Deleted 24 block file(s) for variable 'temperature'.\n"
                "Done, 48 block file(s) deleted total.\n"
            ),
            result.stderr)
        expected_file_names = sorted(['.zarray', '.zattrs'])
        self.assertEqual(expected_file_names,
                         sorted(os.listdir('test.zarr/precipitation')))
        self.assertEqual(expected_file_names,
                         sorted(os.listdir('test.zarr/temperature')))
        ds = xr.open_zarr('test.zarr')
        assert_cube(ds)
        self.assertIn('precipitation', ds)
        self.assertEqual((3, 180, 360), ds.precipitation.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.precipitation.dims)
        self.assertIn('temperature', ds)
        self.assertEqual((3, 180, 360), ds.temperature.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.temperature.dims)

    def test_delete_block_file(self):
        actual_message = None

        def monitor(message, level):
            nonlocal actual_message
            actual_message = message

        actual_message = None
        ok = _delete_block_file(self.TEST_CUBE,
                                'precipitation', (0, 3, 76),
                                True, monitor=monitor)
        self.assertFalse(ok)
        file_path = os.path.join(self.TEST_CUBE,
                                 'precipitation', '0.3.76')
        dir_path = os.path.join(self.TEST_CUBE,
                                'precipitation', '0', '3', '76')
        self.assertEqual(f"Could find neither block file "
                         f"{file_path} nor "
                         f"{dir_path}", actual_message)

        block_file = os.path.join(self.TEST_CUBE,
                                  'precipitation', '1.1.0')

        actual_message = None
        ok = _delete_block_file(self.TEST_CUBE,
                                'precipitation', (1, 1, 0),
                                True, monitor=monitor)
        self.assertTrue(ok)
        self.assertEqual(f'Deleting chunk file {block_file}',
                         actual_message)

        actual_message = None
        ok = _delete_block_file(self.TEST_CUBE,
                                'precipitation', (1, 1, 0),
                                False, monitor=monitor)
        self.assertTrue(ok)
        self.assertEqual(f'Deleting chunk file {block_file}',
                         actual_message)
        self.assertFalse(os.path.exists(block_file))

        if sys.platform == 'win32':
            block_file = os.path.join(self.TEST_CUBE,
                                      'precipitation', '1.1.1')
            # Open block, so we cannot delete (Windows only)
            # noinspection PyUnusedLocal
            with open(block_file, 'wb') as fp:
                actual_message = None
                ok = _delete_block_file(self.TEST_CUBE,
                                        'precipitation', (1, 1, 1),
                                        False, monitor=monitor)
                self.assertFalse(ok)
                self.assertIsNotNone(actual_message)
                # noinspection PyUnresolvedReferences
                self.assertTrue(actual_message.startswith(
                    f'Failed to delete block file {block_file}: '
                ))


# test/cli/test_rectify.py

import os
from typing import List
import xarray as xr
from test.cli.helpers import CliDataTest, TEST_ZARR_DIR
from xcube.core.verify import assert_cube


class RectifyTest(CliDataTest):

    def outputs(self) -> List[str]:
        return ['out.zarr']

    def test_rectify_without_vars(self):
        """Test that rectify selects all variables when --var is not given."""

        # For now, specify the image geometry explicitly with --size, --point,
        # and --res to avoid triggering an "invalid y_min" ValueError when
        # ImageGeom tries to determine it automatically. Once Issue #303 has
        # been fixed, these options can be omitted.

        result = self.invoke_cli(['rectify',
                                  '--verbose',
                                  '--size', '508,253',
                                  '--point', '-179.5,-89.5',
                                  '--res', '0.7071067811865475',
                                  '--crs', 'EPSG:4326',
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('Opening dataset from \'test.zarr\'...\n'
                         'Rectifying...\n'
                         'Writing rectified dataset to \'out.zarr\'...\n'
                         'Done.\n',
                         result.stderr)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('precipitation', ds)
        self.assertIn('temperature', ds)
        self.assertIn('soil_moisture', ds)

    def test_rectify_multiple_comma_separated_vars(self):
        """Test that rectify selects the desired variables when
        multiple --var options, some with multiple comma-separated
        variable names as an argument, are passed."""

        # For now, specify the image geometry explicitly with --size, --point,
        # and --res to avoid triggering an "invalid y_min" ValueError when
        # ImageGeom tries to determine it automatically. Once Issue #303 has
        # been fixed, these options can be omitted.

        result = self.invoke_cli(['rectify',
                                  '--verbose',
                                  '--size', '508,253',
                                  '--point', '-179.5,-89.5',
                                  '--res', '0.7071067811865475',
                                  '--crs', 'EPSG:4326',
                                  '--var', 'precipitation,temperature',
                                  '--var', 'soil_moisture',
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('Opening dataset from \'test.zarr\'...\n'
                         'Rectifying...\n'
                         'Writing rectified dataset to \'out.zarr\'...\n'
                         'Done.\n',
                         result.stderr)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('precipitation', ds)
        self.assertIn('temperature', ds)
        self.assertIn('soil_moisture', ds)


# test/cli/test_resample.py

import os.path
from typing import List

import xarray as xr

from test.cli.helpers import CliTest, CliDataTest, TEST_ZARR_DIR
from xcube.core.verify import assert_cube


class ResampleTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['resample', '--help'])
        self.assertEqual(0, result.exit_code)


class ResampleDataTest(CliDataTest):
    def outputs(self) -> List[str]:
        return ['out.zarr', 'resampled.zarr']

    def test_all_defaults(self):
        result = self.invoke_cli(['resample', '-v', TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertEqual("Opening cube from 'test.zarr'...\n"
                         "Resampling...\n"
                         "Writing resampled cube to 'out.zarr'...\n"
                         "Done.\n",
                         result.stderr)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('precipitation_mean', ds)
        self.assertIn('temperature_mean', ds)
        self.assertIn('soil_moisture_mean', ds)

    def test_with_output(self):
        result = self.invoke_cli(['resample', TEST_ZARR_DIR, '--output', 'resampled.zarr'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir('resampled.zarr'))
        ds = xr.open_zarr('resampled.zarr')
        assert_cube(ds)
        self.assertIn('precipitation_mean', ds)
        self.assertIn('temperature_mean', ds)
        self.assertIn('soil_moisture_mean', ds)

    def test_with_vars(self):
        result = self.invoke_cli(['resample', TEST_ZARR_DIR, '--vars', 'temperature,precipitation'])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('precipitation_mean', ds)
        self.assertIn('temperature_mean', ds)
        self.assertNotIn('soil_moisture_mean', ds)

    def test_downsample_with_multiple_methods(self):
        result = self.invoke_cli(['resample',
                                  '--variables', 'temperature',
                                  '-F', 'all',
                                  '-M', 'mean',
                                  '-M', 'count',
                                  '-M', 'prod',
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        self.assertIn('temperature_mean', ds)
        self.assertIn('temperature_count', ds)
        self.assertIn('temperature_prod', ds)

    def test_upsample_with_multiple_methods(self):
        result = self.invoke_cli(['resample',
                                  '--variables', 'temperature',
                                  '-F', '12H',
                                  '-T', '6H',
                                  # '-K', 'quadratic',
                                  # '-M', 'interpolate',
                                  '-M', 'nearest',
                                  TEST_ZARR_DIR])
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir('out.zarr'))
        ds = xr.open_zarr('out.zarr')
        assert_cube(ds)
        # self.assertIn('temperature_interpolate', ds)
        self.assertIn('temperature_nearest', ds)


# test/cli/test_serve.py

import os
import os.path
from typing import Mapping

from test.cli.helpers import CliDataTest
from xcube.cli.serve import BASE_ENV_VAR
from xcube.cli.serve import CONFIG_ENV_VAR
from xcube.cli.serve import VIEWER_ENV_VAR
from xcube.cli.serve import main


class ServerCliTest(CliDataTest):
    def test_config(self):
        result = self.invoke_cli(["serve", "--config", "x.yml", "pippo.zarr"])
        self.assertEqual('Error: CONFIG and CUBES cannot be used at the same time.\n',
                         result.stderr[-58:])
        self.assertEqual(1, result.exit_code)

        with ChangeEnv({CONFIG_ENV_VAR: "x.yml"}):
            result = self.invoke_cli(["serve"])
            self.assertEqual('Error: Configuration file not found: x.yml\n',
                             result.stderr[-44:])
            self.assertEqual(1, result.exit_code)

    def test_base_dir(self):
        config_path = os.path.join(os.path.dirname(__file__),
                                   '..', '..', 'examples', 'serve', 'demo', 'config.yml')

        with ChangeEnv({BASE_ENV_VAR: None}):
            result = self.invoke_cli(["serve", "--config", config_path, "--base-dir", "pippo/gnarz"])
            self.assertEqual('Error: Base directory not found: pippo/gnarz\n',
                             result.stderr[-45:])
            self.assertEqual(1, result.exit_code)

        with ChangeEnv({BASE_ENV_VAR: 'pippo/configs'}):
            result = self.invoke_cli(["serve", "--config", config_path])
            self.assertEqual('Error: Base directory not found: pippo/configs\n',
                             result.stderr[-47:])
            self.assertEqual(1, result.exit_code)

    def test_show(self):
        config_path = os.path.join(os.path.dirname(__file__),
                                   '..', '..', 'examples', 'serve', 'demo', 'config.yml')

        with ChangeEnv({VIEWER_ENV_VAR: None}):
            result = self.invoke_cli(["serve", "--show", "--config", config_path])
            self.assertEqual(f'Error: Option "--show": In order to run the viewer, '
                             f'set environment variable {VIEWER_ENV_VAR} to a valid '
                             f'xcube-viewer deployment or build directory.\n',
                             result.stderr[-150:])
            self.assertEqual(2, result.exit_code)

        with ChangeEnv({VIEWER_ENV_VAR: "pippo/viewer"}):
            result = self.invoke_cli(["serve", "--show", "--config", config_path])
            self.assertEqual(f'Error: Option "--show": Viewer path set by environment '
                             f'variable {VIEWER_ENV_VAR} '
                             f'must be a directory: pippo/viewer\n',
                             result.stderr[-116:])
            self.assertEqual(2, result.exit_code)

    def test_help(self):
        with self.assertRaises(SystemExit):
            main(args=["--help"])


class ChangeEnv:
    def __init__(self, env=None, **env_kwargs):
        self._new_env = dict(env) if env else dict()
        self._new_env.update(**env_kwargs)
        self._old_env = None

    def __enter__(self) -> Mapping[str, str]:
        self._old_env = dict()
        for k, v in self._new_env.items():
            self._old_env[k] = os.environ.get(k)
            if v is not None:
                os.environ[k] = str(v)
            elif k in os.environ:
                del os.environ[k]
        return os.environ

    def __exit__(self, *error):
        for k, v in self._old_env.items():
            if v is not None:
                os.environ[k] = v
            elif k in os.environ:
                del os.environ[k]


# test/cli/test_serve2.py

from test.cli.helpers import CliTest


class ServerCliTest(CliTest):
    def test_help(self):
        result = self.invoke_cli(["serve2", "--help"])
        self.assertEqual(0, result.exit_code)

    def test_update_after(self):
        result = self.invoke_cli(["serve2",
                                  "--update-after", "0.1",
                                  "--stop-after", "0.2"])
        self.assertEqual(0, result.exit_code)

    def test_commands(self):
        result = self.invoke_cli(["serve2", "list", "apis"])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(["serve2", "show", "openapi"])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(["serve2", "show", "config"])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(["serve2", "show", "configschema"])
        self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(["serve2", "list", "routes"])
        self.assertEqual(1, result.exit_code)

    def test_commands_with_format(self):
        for f in ["yaml", "json"]:
            result = self.invoke_cli(["serve2", "show", "openapi"] + [f])
            self.assertEqual(0, result.exit_code)

            result = self.invoke_cli(["serve2", "show", "config"] + [f])
            self.assertEqual(0, result.exit_code)

            result = self.invoke_cli(["serve2", "show", "configschema"] + [f])
            self.assertEqual(0, result.exit_code)

        result = self.invoke_cli(["serve2", "show", "configschema", "nc"])
        self.assertEqual(1, result.exit_code)


# test/cli/test_tile.py

import os.path
from typing import List, Tuple

from test.cli.helpers import CliDataTest
from test.cli.helpers import CliTest
from test.cli.helpers import TEST_NC_FILE, TEST_ZARR_DIR


class TileTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['tile', '--help'])
        self.assertEqual(0, result.exit_code)


class LevelDataTest(CliDataTest):
    TEST_OUTPUT = "out.tiles"

    def outputs(self) -> List[str]:
        return [LevelDataTest.TEST_OUTPUT, 'my.tiles', 'config.yml']

    def time_periods(self) -> int:
        return 2

    def test_all_defaults(self):
        result = self.invoke_cli(['tile', TEST_ZARR_DIR])
        self._assert_result_ok(result, LevelDataTest.TEST_OUTPUT, [
            'out.tiles/precipitation/0/0/0/0.png',
            'out.tiles/precipitation/0/0/0/1.png',
            'out.tiles/precipitation/0/tilemapresource.xml',
            'out.tiles/precipitation/1/0/0/0.png',
            'out.tiles/precipitation/1/0/0/1.png',
            'out.tiles/precipitation/1/tilemapresource.xml',
            'out.tiles/precipitation/metadata.json',
            'out.tiles/soil_moisture/0/0/0/0.png',
            'out.tiles/soil_moisture/0/0/0/1.png',
            'out.tiles/soil_moisture/0/tilemapresource.xml',
            'out.tiles/soil_moisture/1/0/0/0.png',
            'out.tiles/soil_moisture/1/0/0/1.png',
            'out.tiles/soil_moisture/1/tilemapresource.xml',
            'out.tiles/soil_moisture/metadata.json',
            'out.tiles/temperature/0/0/0/0.png',
            'out.tiles/temperature/0/0/0/1.png',
            'out.tiles/temperature/0/tilemapresource.xml',
            'out.tiles/temperature/1/0/0/0.png',
            'out.tiles/temperature/1/0/0/1.png',
            'out.tiles/temperature/1/tilemapresource.xml',
            'out.tiles/temperature/metadata.json'
        ])

    def test_with_output(self):
        result = self.invoke_cli(['tile', TEST_ZARR_DIR, '--output', 'my.tiles', '-vvv'])
        self._assert_result_ok(result, 'my.tiles', [
            'my.tiles/precipitation/0/0/0/0.png',
            'my.tiles/precipitation/0/0/0/1.png',
            'my.tiles/precipitation/0/tilemapresource.xml',
            'my.tiles/precipitation/1/0/0/0.png',
            'my.tiles/precipitation/1/0/0/1.png',
            'my.tiles/precipitation/1/tilemapresource.xml',
            'my.tiles/precipitation/metadata.json',
            'my.tiles/soil_moisture/0/0/0/0.png',
            'my.tiles/soil_moisture/0/0/0/1.png',
            'my.tiles/soil_moisture/0/tilemapresource.xml',
            'my.tiles/soil_moisture/1/0/0/0.png',
            'my.tiles/soil_moisture/1/0/0/1.png',
            'my.tiles/soil_moisture/1/tilemapresource.xml',
            'my.tiles/soil_moisture/metadata.json',
            'my.tiles/temperature/0/0/0/0.png',
            'my.tiles/temperature/0/0/0/1.png',
            'my.tiles/temperature/0/tilemapresource.xml',
            'my.tiles/temperature/1/0/0/0.png',
            'my.tiles/temperature/1/0/0/1.png',
            'my.tiles/temperature/1/tilemapresource.xml',
            'my.tiles/temperature/metadata.json'
        ])

    def test_with_tile_size(self):
        result = self.invoke_cli(['tile', TEST_ZARR_DIR, '-t', '90'])
        self._assert_result_ok(result, LevelDataTest.TEST_OUTPUT, [
            'out.tiles/precipitation/0/0/0/0.png',
            'out.tiles/precipitation/0/0/1/0.png',
            'out.tiles/precipitation/0/1/0/0.png',
            'out.tiles/precipitation/0/1/0/1.png',
            'out.tiles/precipitation/0/1/1/0.png',
            'out.tiles/precipitation/0/1/1/1.png',
            'out.tiles/precipitation/0/1/2/0.png',
            'out.tiles/precipitation/0/1/2/1.png',
            'out.tiles/precipitation/0/1/3/0.png',
            'out.tiles/precipitation/0/1/3/1.png',
            'out.tiles/precipitation/0/tilemapresource.xml',
            'out.tiles/precipitation/1/0/0/0.png',
            'out.tiles/precipitation/1/0/1/0.png',
            'out.tiles/precipitation/1/1/0/0.png',
            'out.tiles/precipitation/1/1/0/1.png',
            'out.tiles/precipitation/1/1/1/0.png',
            'out.tiles/precipitation/1/1/1/1.png',
            'out.tiles/precipitation/1/1/2/0.png',
            'out.tiles/precipitation/1/1/2/1.png',
            'out.tiles/precipitation/1/1/3/0.png',
            'out.tiles/precipitation/1/1/3/1.png',
            'out.tiles/precipitation/1/tilemapresource.xml',
            'out.tiles/precipitation/metadata.json',
            'out.tiles/soil_moisture/0/0/0/0.png',
            'out.tiles/soil_moisture/0/0/1/0.png',
            'out.tiles/soil_moisture/0/1/0/0.png',
            'out.tiles/soil_moisture/0/1/0/1.png',
            'out.tiles/soil_moisture/0/1/1/0.png',
            'out.tiles/soil_moisture/0/1/1/1.png',
            'out.tiles/soil_moisture/0/1/2/0.png',
            'out.tiles/soil_moisture/0/1/2/1.png',
            'out.tiles/soil_moisture/0/1/3/0.png',
            'out.tiles/soil_moisture/0/1/3/1.png',
            'out.tiles/soil_moisture/0/tilemapresource.xml',
            'out.tiles/soil_moisture/1/0/0/0.png',
            'out.tiles/soil_moisture/1/0/1/0.png',
            'out.tiles/soil_moisture/1/1/0/0.png',
            'out.tiles/soil_moisture/1/1/0/1.png',
            'out.tiles/soil_moisture/1/1/1/0.png',
            'out.tiles/soil_moisture/1/1/1/1.png',
            'out.tiles/soil_moisture/1/1/2/0.png',
            'out.tiles/soil_moisture/1/1/2/1.png',
            'out.tiles/soil_moisture/1/1/3/0.png',
            'out.tiles/soil_moisture/1/1/3/1.png',
            'out.tiles/soil_moisture/1/tilemapresource.xml',
            'out.tiles/soil_moisture/metadata.json',
            'out.tiles/temperature/0/0/0/0.png',
            'out.tiles/temperature/0/0/1/0.png',
            'out.tiles/temperature/0/1/0/0.png',
            'out.tiles/temperature/0/1/0/1.png',
            'out.tiles/temperature/0/1/1/0.png',
            'out.tiles/temperature/0/1/1/1.png',
            'out.tiles/temperature/0/1/2/0.png',
            'out.tiles/temperature/0/1/2/1.png',
            'out.tiles/temperature/0/1/3/0.png',
            'out.tiles/temperature/0/1/3/1.png',
            'out.tiles/temperature/0/tilemapresource.xml',
            'out.tiles/temperature/1/0/0/0.png',
            'out.tiles/temperature/1/0/1/0.png',
            'out.tiles/temperature/1/1/0/0.png',
            'out.tiles/temperature/1/1/0/1.png',
            'out.tiles/temperature/1/1/1/0.png',
            'out.tiles/temperature/1/1/1/1.png',
            'out.tiles/temperature/1/1/2/0.png',
            'out.tiles/temperature/1/1/2/1.png',
            'out.tiles/temperature/1/1/3/0.png',
            'out.tiles/temperature/1/1/3/1.png',
            'out.tiles/temperature/1/tilemapresource.xml',
            'out.tiles/temperature/metadata.json'
        ])

    def _assert_result_ok(self, result, output_path: str, actual_file_listing: List[Tuple]):
        self.assertEqual(0, result.exit_code)
        # self.assertRegex(result.stdout, message_regex)
        self.assertTrue(os.path.isdir(output_path))

        expected_file_listing = []
        for root, dirs, files in os.walk(output_path, topdown=False):
            for name in files:
                expected_file_listing.append(os.path.join(root, name).replace('\\', '/'))
        expected_file_listing.sort()

        self.assertEqual(expected_file_listing, actual_file_listing)

    def test_invalid_inputs(self):
        result = self.invoke_cli(["tile",
                                  "-t", "a45",
                                  TEST_ZARR_DIR,
                                  ])
        self._assert_result_not_ok(result,
                                   "Error\\: Invalid tile sizes in TILE_SIZE found: "
                                   "invalid literal for int\\(\\) with base 10\\: 'a45'\n")

        result = self.invoke_cli(["tile",
                                  "-t", "-3",
                                  TEST_NC_FILE,
                                  ])
        self._assert_result_not_ok(result,
                                   "Error\\: Invalid tile sizes in TILE_SIZE found\\: "
                                   "all items must be positive integer numbers\n")

        result = self.invoke_cli(["tile",
                                  "-t", "45,45,45",
                                  TEST_NC_FILE])
        self._assert_result_not_ok(result,
                                   "Error\\: TILE_SIZE must have 2 tile sizes separated by ','\n")

    def _assert_result_not_ok(self, result, message_regex: str):
        self.assertEqual(1, result.exit_code)
        self.assertRegex(result.stderr, message_regex)


# test/cli/test_vars2dim.py

import os

import xarray as xr

from test.cli.helpers import CliDataTest, TEST_ZARR_DIR


class Vars2DimTest(CliDataTest):
    TEST_OUTPUT = "test-vars2dim.zarr"

    def outputs(self):
        return [self.TEST_OUTPUT]

    def test_vars2dim(self):
        result = self.invoke_cli(["vars2dim", TEST_ZARR_DIR])

        output_path = self.TEST_OUTPUT
        self.assertEqual(0, result.exit_code)
        self.assertTrue(os.path.isdir(output_path))

        ds = xr.open_zarr(output_path)
        self.assertIn("var", ds.dims)
        self.assertEqual(3, ds.dims["var"])
        self.assertIn("var", ds.coords)
        self.assertIn("data", ds.data_vars)
        var_names = ds["var"]
        self.assertEqual(("var",), var_names.dims)
        self.assertTrue(hasattr(var_names, "encoding"))
        self.assertEqual(3, len(var_names))
        self.assertIn("precipitation", str(var_names[0]))
        self.assertIn("soil_moisture", str(var_names[1]))
        self.assertIn("temperature", str(var_names[2]))


# test/cli/test_verify.py

import numpy as np
import xarray as xr

from test.cli.helpers import CliTest
from xcube.core.dsio import rimraf
from xcube.core.dsio import write_cube
from xcube.core.new import new_cube


class VerifyTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['verify', '--help'])
        self.assertEqual(0, result.exit_code)


class VerifyDataTest(CliTest):
    TEST_CUBE = "test.zarr"

    def setUp(self) -> None:
        rimraf(self.TEST_CUBE)

    def tearDown(self) -> None:
        rimraf(self.TEST_CUBE)

    def test_verify_ok(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
        result = self.invoke_cli(['verify', self.TEST_CUBE])
        self.assertEqual(0, result.exit_code)
        self.assertEqual("INPUT is a valid cube.\n",
                         result.stdout)

    def test_verify_failure(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        cube["chl"] = xr.DataArray(np.random.rand(cube.dims["lat"], cube.dims["lon"]),
                                   dims=("lat", "lon"),
                                   coords=dict(lat=cube.lat, lon=cube.lon))
        write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)

        result = self.invoke_cli(['verify', self.TEST_CUBE])
        self.assertEqual(3, result.exit_code)
        self.assertEqual("INPUT is not a valid cube due to the following reasons:\n"
                         "- dimensions of data variable 'chl' must be ('time', ..., 'lat', 'lon'),"
                         " but were ('lat', 'lon') for 'chl'\n"
                         "- dimensions of all data variables must be same, but found ('lat', 'lon')"
                         " for 'chl' and ('time', 'lat', 'lon') for 'precipitation'\n"
                         "- all data variables must have same chunk sizes, but found ((90, 90), (360,))"
                         " for 'chl' and ((3, 2), (90, 90), (180, 180)) for 'precipitation'\n",
                         result.stdout)


# test/cli/test_versions.py

from test.cli.helpers import CliTest
from xcube.util.temp import new_temp_file


class VersionsTest(CliTest):

    def test_help_option(self):
        result = self.invoke_cli(['versions', '--help'])
        self.assertEqual(0, result.exit_code)

    def test_no_args(self):
        result = self.invoke_cli(['versions'])
        self.assertEqual(0, result.exit_code)
        self.assertYaml(result.stdout)

    def test_format(self):
        result = self.invoke_cli(['versions', '-f', 'json'])
        self.assertEqual(0, result.exit_code)
        self.assertJson(result.stdout)

        result = self.invoke_cli(['versions', '-f', 'yaml'])
        self.assertEqual(0, result.exit_code)
        self.assertYaml(result.stdout)

    def test_output(self):
        _, output_path = new_temp_file(suffix='.json')
        result = self.invoke_cli(['versions', '-o', output_path])
        self.assertEqual(0, result.exit_code)
        with open(output_path) as fp:
            text = fp.read()
        self.assertJson(text)

        _, output_path = new_temp_file(suffix='.yml')
        result = self.invoke_cli(['versions', '-o', output_path])
        self.assertEqual(0, result.exit_code)
        with open(output_path) as fp:
            text = fp.read()
        self.assertYaml(text)

    def assertJson(self, json_result: str):
        self.assertIn('{', json_result)
        self.assertIn(':', json_result)
        self.assertIn('}', json_result)

    def assertYaml(self, json_result: str):
        self.assertNotIn('{', json_result)
        self.assertIn(':', json_result)
        self.assertNotIn('}', json_result)


# test/core/diagnosticstore.py

import time
from collections.abc import MutableMapping
from types import MethodType
from typing import TypeVar, Iterator, List, Callable, Any, Tuple

_KT = TypeVar('_KT')
_VT = TypeVar('_VT')
_T_co = TypeVar('_T_co')
_VT_co = TypeVar('_VT_co')


class DiagnosticStore(MutableMapping):

    def __init__(self,
                 delegate: MutableMapping,
                 observer: Callable[[int, float, str, List[Tuple[str, Any]]], None] = None):
        self._delegate = delegate
        self._observer = observer or logging_observer()
        self._counter = 0
        self._add_optional_method('listdir', ['path'])
        self._add_optional_method('rmdir', ['path'])
        self._add_optional_method('rename', ['from_path', 'to_path'])

    def _add_optional_method(self, method_name: str, arg_names: List[str]):
        if hasattr(self._delegate, method_name):
            def method(_self, *args) -> List[str]:
                return _self.call_and_notify(method_name, *[(arg_names[i], args[i]) for i in range(len(args))])

            setattr(self, method_name, MethodType(method, self))

    def call_and_notify(self, method_name: str, *args):
        method = getattr(self._delegate, method_name)

        t0 = time.perf_counter()
        result = method(*(arg[1] for arg in args))
        t1 = time.perf_counter()

        self._counter += 1
        self._observer(self._counter, t1 - t0, method_name, *args)

        return result

    def __contains__(self, k: _KT) -> bool:
        return self.call_and_notify('__contains__', ('k', k))

    def __setitem__(self, k: _KT, v: _VT) -> None:
        return self.call_and_notify('__setitem__', ('k', k), ('v', v))

    def __delitem__(self, k: _KT) -> None:
        return self.call_and_notify('__delitem__', ('k', k))

    def __getitem__(self, k: _KT) -> _VT_co:
        return self.call_and_notify('__getitem__', ('k', k))

    def __len__(self) -> int:
        return self.call_and_notify('__len__')

    def __iter__(self) -> Iterator[_T_co]:
        return self.call_and_notify('__iter__')


def logging_observer(logger_name=None, log_path=None, log_all=False):
    import logging

    observer_logger = logging.getLogger(logger_name or 'diagnosticstore')

    if log_all:
        logger = logging.getLogger()
    else:
        logger = observer_logger

    logger.setLevel(logging.DEBUG)
    handler = logging.FileHandler(log_path or 'diagnosticstore.log')
    handler.setFormatter(logging.Formatter('%(asctime)s: %(levelname)s: %(name)s: %(message)s'))
    logger.addHandler(handler)

    def observer(counter, time_needed, method_name, *args):
        msg = f'call #{counter}: {method_name}('
        msg += ', '.join(map(lambda x: f'{x[0]}={repr(x[1])}', args))
        msg += f'), took {int(1000 * time_needed)} ms'
        observer_logger.info(msg)

    return observer


# test/core/test_ancvar.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.ancvar import find_ancillary_var_names


class FindAncillaryVarNamesTest(unittest.TestCase):
    @classmethod
    def _new_array(self, dims=None, **attrs):
        dims = dims or [('time', 3), ('lat', 3), ('lon', 3)]
        s = 1
        for d in dims:
            s *= d[1]
        return xr.DataArray(np.linspace(0, 1, s).reshape([s for _, s in dims]), dims=([n for n, _ in dims]),
                            attrs=attrs)

    def test_strict_cf_convention(self):
        a = self._new_array(standard_name='a', ancillary_variables='a_stdev b c')
        a_stdev = self._new_array(standard_name='a standard_error')
        a_uncert = self._new_array(standard_name='a standard_error')
        a_count = self._new_array(standard_name='a number_of_observations')
        b = self._new_array(standard_name='b')
        b_stdev = self._new_array(standard_name='b standard_error')
        c = self._new_array(standard_name='c')

        ds = xr.Dataset(dict(a=a))
        self.assertEqual({}, find_ancillary_var_names(ds, 'a'))

        ds = xr.Dataset(dict(a=a, a_stdev=a_stdev, a_uncert=a_uncert, a_count=a_count, b=b, b_stdev=b_stdev, c=c))
        self.assertEqual({'': {'c', 'b'}, 'standard_error': {'a_stdev'}},
                         find_ancillary_var_names(ds, 'a'))

    def test_less_strict_cf_convention(self):
        a = self._new_array(standard_name='a')
        a_stdev = self._new_array(standard_name='a standard_error')
        a_uncert = self._new_array(standard_name='a standard_error')
        a_count = self._new_array(standard_name='a number_of_observations')
        b = self._new_array(standard_name='b')
        b_stdev = self._new_array(standard_name='b standard_error')

        ds = xr.Dataset(dict(a=a))
        self.assertEqual({}, find_ancillary_var_names(ds, 'a'))

        ds = xr.Dataset(dict(a=a, a_stdev=a_stdev, a_uncert=a_uncert, a_count=a_count, b=b, b_stdev=b_stdev))
        self.assertEqual({'number_of_observations': {'a_count'},
                          'standard_error': {'a_stdev', 'a_uncert'}},
                         find_ancillary_var_names(ds, 'a'))

    def test_xcube_prefix_convention(self):
        a = self._new_array()
        a_std = self._new_array()
        a_count = self._new_array()
        b = self._new_array()
        b_std = self._new_array()

        ds = xr.Dataset(dict(a=a))
        self.assertEqual({}, find_ancillary_var_names(ds, 'a'))

        ds = xr.Dataset(dict(a=a, a_std=a_std, a_count=a_count, b=b, b_std=b_std))
        self.assertEqual({'number_of_observations': {'a_count'},
                          'standard_error': {'a_std'}},
                         find_ancillary_var_names(ds, 'a'))


# test/core/test_chunk.py

import collections
import unittest

import numpy as np

from test.sampledata import new_test_dataset
from xcube.core.chunk import chunk_dataset
from xcube.core.chunk import compute_chunk_slices
from xcube.core.chunk import get_empty_dataset_chunks


class ChunkDatasetTest(unittest.TestCase):

    def test_chunk_dataset(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02",
                                    "2010-01-03", "2010-01-04",
                                    "2010-01-05"],
                                   precipitation=0.4, temperature=275.2)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=dict(time=1,
                                                         lat=10, lon=20),
                                        format_name="zarr")
        self.assertEqual({'chunks': (1, 10, 20)},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({'chunks': (1, 10, 20)},
                         chunked_dataset.temperature.encoding)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=dict(time=1,
                                                         lat=20, lon=40),
                                        format_name="netcdf4")
        self.assertEqual({'chunksizes': (1, 20, 40)},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({'chunksizes': (1, 20, 40)},
                         chunked_dataset.temperature.encoding)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=dict(time=1,
                                                         lat=20, lon=40))
        self.assertEqual({}, chunked_dataset.precipitation.encoding)
        self.assertEqual({}, chunked_dataset.temperature.encoding)

        dataset = dataset.chunk(dict(time=2, lat=10, lon=20))

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=None,
                                        format_name="zarr")
        self.assertEqual({}, chunked_dataset.precipitation.encoding)
        self.assertEqual({}, chunked_dataset.temperature.encoding)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes={},
                                        format_name="zarr")
        self.assertEqual({'chunks': (2, 10, 20)},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({'chunks': (2, 10, 20)},
                         chunked_dataset.temperature.encoding)

        chunked_dataset = chunk_dataset(dataset,
                                        chunk_sizes=dict(time=1),
                                        format_name="zarr")
        self.assertEqual({'chunks': (1, 10, 20)},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({'chunks': (1, 10, 20)},
                         chunked_dataset.temperature.encoding)

    def test_unchunk_dataset(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02",
                                    "2010-01-03", "2010-01-04",
                                    "2010-01-05"],
                                   precipitation=0.4,
                                   temperature=275.2)

        for var in dataset.data_vars.values():
            var.encoding.update({"chunks": (5, 180, 360),
                                 "_FillValue": -999.0})

        chunked_dataset = chunk_dataset(dataset, format_name="zarr")
        self.assertEqual({"_FillValue": -999.0},
                         chunked_dataset.precipitation.encoding)
        self.assertEqual({"_FillValue": -999.0},
                         chunked_dataset.temperature.encoding)


class GetEmptyDatasetChunksTest(unittest.TestCase):

    def test_not_chunked(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02"],
                                   precipitation=0.4,
                                   temperature=275.2)
        empty_dataset_chunks = get_empty_dataset_chunks(dataset)
        self.assertIsInstance(empty_dataset_chunks, collections.Iterator)
        self.assertFalse(isinstance(empty_dataset_chunks, (list, tuple)))
        self.assertEqual([('precipitation', ()),
                          ('temperature', ())],
                         [(v, tuple(c)) for v, c in empty_dataset_chunks])

    def test_non_empty(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02"],
                                   precipitation=0.4,
                                   temperature=275.2) \
            .chunk(dict(time=1, lat=90, lon=90))
        empty_dataset_chunks = get_empty_dataset_chunks(dataset)
        self.assertIsInstance(empty_dataset_chunks, collections.Iterator)
        self.assertFalse(isinstance(empty_dataset_chunks, (list, tuple)))
        self.assertEqual([('precipitation', ()),
                          ('temperature', ())],
                         [(v, tuple(c)) for v, c in empty_dataset_chunks])

    def test_all_empty(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02"],
                                   precipitation=np.nan,
                                   temperature=np.nan) \
            .chunk(dict(time=1, lat=90, lon=90))
        empty_dataset_chunks = get_empty_dataset_chunks(dataset)
        self.assertIsInstance(empty_dataset_chunks, collections.Iterator)
        self.assertFalse(isinstance(empty_dataset_chunks, (list, tuple)))
        self.assertEqual([('precipitation', ((0, 0, 0),
                                             (0, 0, 1),
                                             (0, 0, 2),
                                             (0, 0, 3),
                                             (0, 1, 0),
                                             (0, 1, 1),
                                             (0, 1, 2),
                                             (0, 1, 3),
                                             (1, 0, 0),
                                             (1, 0, 1),
                                             (1, 0, 2),
                                             (1, 0, 3),
                                             (1, 1, 0),
                                             (1, 1, 1),
                                             (1, 1, 2),
                                             (1, 1, 3))),
                          ('temperature', ((0, 0, 0),
                                           (0, 0, 1),
                                           (0, 0, 2),
                                           (0, 0, 3),
                                           (0, 1, 0),
                                           (0, 1, 1),
                                           (0, 1, 2),
                                           (0, 1, 3),
                                           (1, 0, 0),
                                           (1, 0, 1),
                                           (1, 0, 2),
                                           (1, 0, 3),
                                           (1, 1, 0),
                                           (1, 1, 1),
                                           (1, 1, 2),
                                           (1, 1, 3)))],
                         [(v, tuple(c)) for v, c in empty_dataset_chunks])


class ComputeChunkSlicesTest(unittest.TestCase):

    def test_compute_chunk_slices(self):
        chunk_slices = compute_chunk_slices(((1, 1),
                                             (90, 90), (90, 90, 90, 90)))
        self.assertEqual([((0, 0, 0), ((0, 1), (0, 90), (0, 90))),
                          ((0, 0, 1), ((0, 1), (0, 90), (90, 180))),
                          ((0, 0, 2), ((0, 1), (0, 90), (180, 270))),
                          ((0, 0, 3), ((0, 1), (0, 90), (270, 360))),
                          ((0, 1, 0), ((0, 1), (90, 180), (0, 90))),
                          ((0, 1, 1), ((0, 1), (90, 180), (90, 180))),
                          ((0, 1, 2), ((0, 1), (90, 180), (180, 270))),
                          ((0, 1, 3), ((0, 1), (90, 180), (270, 360))),
                          ((1, 0, 0), ((1, 2), (0, 90), (0, 90))),
                          ((1, 0, 1), ((1, 2), (0, 90), (90, 180))),
                          ((1, 0, 2), ((1, 2), (0, 90), (180, 270))),
                          ((1, 0, 3), ((1, 2), (0, 90), (270, 360))),
                          ((1, 1, 0), ((1, 2), (90, 180), (0, 90))),
                          ((1, 1, 1), ((1, 2), (90, 180), (90, 180))),
                          ((1, 1, 2), ((1, 2), (90, 180), (180, 270))),
                          ((1, 1, 3), ((1, 2), (90, 180), (270, 360)))],
                         list(chunk_slices))


# test/core/test_chunkstore.py

import unittest
import warnings
from typing import Tuple

import numpy as np
import xarray as xr
from zarr.storage import MemoryStore

from xcube.core.chunkstore import ChunkStore
from xcube.core.chunkstore import LoggingStore
from xcube.core.chunkstore import MutableLoggingStore


class ChunkStoreTest(unittest.TestCase):
    def test_chunk_store(self):
        self._test_chunk_store(trace_store_calls=False)

    def test_chunk_store_with_tracing(self):
        self._test_chunk_store(trace_store_calls=True)

    def _test_chunk_store(self, trace_store_calls: bool):
        index_var = gen_index_var(dims=('time', 'lat', 'lon'),
                                  shape=(4, 8, 16),
                                  chunks=(2, 4, 8),
                                  trace_store_calls=trace_store_calls)
        self.assertIsNotNone(index_var)
        self.assertEqual((4, 8, 16), index_var.shape)
        self.assertEqual(((2, 2), (4, 4), (8, 8)), index_var.chunks)
        self.assertEqual(('time', 'lat', 'lon'), index_var.dims)

        visited_indexes = set()

        def index_var_ufunc(index_var_):
            if index_var_.size < 6:
                warnings.warn(f"weird variable of size {index_var_.size} received!")
                return
            nonlocal visited_indexes
            index = tuple(map(int, index_var_.ravel()[0:6]))
            visited_indexes.add(index)
            return index_var_

        result = xr.apply_ufunc(index_var_ufunc, index_var, dask='parallelized', output_dtypes=[index_var.dtype])
        self.assertIsNotNone(result)
        self.assertEqual((4, 8, 16), result.shape)
        self.assertEqual(((2, 2), (4, 4), (8, 8)), result.chunks)
        self.assertEqual(('time', 'lat', 'lon'), result.dims)

        values = result.values
        self.assertEqual((4, 8, 16), values.shape)
        np.testing.assert_array_equal(index_var.values, values)

        print(visited_indexes)

        self.assertEqual(8, len(visited_indexes))
        self.assertEqual({
            (0, 2, 0, 4, 0, 8),
            (2, 4, 0, 4, 0, 8),
            (0, 2, 4, 8, 0, 8),
            (2, 4, 4, 8, 0, 8),
            (0, 2, 0, 4, 8, 16),
            (2, 4, 0, 4, 8, 16),
            (0, 2, 4, 8, 8, 16),
            (2, 4, 4, 8, 8, 16),
        }, visited_indexes)


def gen_index_var(dims, shape, chunks, trace_store_calls: bool = False):
    # noinspection PyUnusedLocal
    def get_chunk(cube_store: ChunkStore, name: str,
                  index: Tuple[int, ...]) -> bytes:
        data = np.zeros(cube_store.chunks, dtype=np.uint64)
        data_view = data.ravel()
        if data_view.base is not data:
            raise ValueError('view expected')
        if data_view.size < cube_store.ndim * 2:
            raise ValueError('size too small')
        for i in range(cube_store.ndim):
            j1 = cube_store.chunks[i] * index[i]
            j2 = j1 + cube_store.chunks[i]
            data_view[2 * i] = j1
            data_view[2 * i + 1] = j2
        return data.tobytes()

    store = ChunkStore(dims, shape, chunks,
                       trace_store_calls=trace_store_calls)
    store.add_lazy_array('__index_var__', '<u8', get_chunk=get_chunk)

    ds = xr.open_zarr(store)
    return ds.__index_var__


class LoggingStoreTest(unittest.TestCase):
    def test_immutable_new(self):
        self.assertReadOk(LoggingStore.new(self.original_store))

    def test_mutable_new(self):
        self.assertWriteOk(LoggingStore.new(self.original_store))

    def test_immutable(self):
        self.assertReadOk(LoggingStore(self.original_store))

    def test_mutable(self):
        self.assertWriteOk(MutableLoggingStore(self.original_store))

    def setUp(self) -> None:
        self.zattrs_value = bytes()
        self.original_store = MemoryStore()
        self.original_store.update({'chl/.zattrs': self.zattrs_value})

    def assertReadOk(self, logging_store: LoggingStore):
        # noinspection PyUnresolvedReferences
        self.assertEqual(['.zattrs'],
                         logging_store.listdir('chl'))
        # noinspection PyUnresolvedReferences
        self.assertEqual(0,
                         logging_store.getsize('chl'))
        self.assertEqual({'chl/.zattrs'},
                         set(logging_store.keys()))
        self.assertEqual(['chl/.zattrs'],
                         list(iter(logging_store)))
        self.assertTrue('chl/.zattrs' in logging_store)
        self.assertEqual(1,
                         len(logging_store))
        self.assertEqual(self.zattrs_value,
                         logging_store.get('chl/.zattrs'))
        # assert original_store not changed
        self.assertEqual({'chl/.zattrs'},
                         set(self.original_store.keys()))

    def assertWriteOk(self, logging_store: MutableLoggingStore):
        zarray_value = bytes()
        logging_store['chl/.zarray'] = zarray_value
        self.assertEqual({'chl/.zattrs',
                          'chl/.zarray'},
                         set(self.original_store.keys()))
        del logging_store['chl/.zarray']
        self.assertEqual({'chl/.zattrs'},
                         set(self.original_store.keys()))


# test/core/test_compute.py

import os
import unittest
from typing import Any, Dict, Tuple

import numpy as np
import xarray as xr

from xcube.core.chunk import chunk_dataset
from xcube.core.compute import CubeFuncOutput
from xcube.core.compute import compute_cube
from xcube.core.compute import compute_dataset
from xcube.core.new import new_cube
from xcube.core.schema import CubeSchema


class ComputeCubeTest(unittest.TestCase):

    def setUp(self) -> None:
        cube = new_cube(width=360,
                        height=180,
                        time_periods=6,
                        variables=dict(analysed_sst=275.3,
                                       analysis_error=2.1))
        cube = chunk_dataset(cube, dict(time=3, lat=90, lon=90))
        self.cube = cube

    def test_without_inputs(self):
        calls = []

        def my_cube_func(input_params: Dict[str, Any] = None,
                         dim_coords: Dict[str, np.ndarray] = None,
                         dim_ranges: Dict[str, Tuple[int, int]] = None) -> CubeFuncOutput:
            nonlocal calls
            calls.append((input_params, dim_coords, dim_ranges))
            lon_range = dim_ranges['lon']
            lat_range = dim_ranges['lat']
            time_range = dim_ranges['time']
            n_lon = lon_range[1] - lon_range[0]
            n_lat = lat_range[1] - lat_range[0]
            n_time = time_range[1] - time_range[0]
            fill_value = input_params['fill_value']
            return np.full((n_time, n_lat, n_lon), fill_value, dtype=np.float64)

        output_cube = compute_cube(my_cube_func,
                                   input_cube_schema=CubeSchema.new(self.cube),
                                   input_params=dict(fill_value=0.74))

        self.assertIsInstance(output_cube, xr.Dataset)
        self.assertIn('output', output_cube.data_vars)
        output_var = output_cube.output
        self.assertEqual(0, len(calls))
        self.assertEqual(('time', 'lat', 'lon'), output_var.dims)
        self.assertEqual((6, 180, 360), output_var.shape)

        values = output_var.values
        self.assertEqual(2 * 2 * 4, len(calls))
        self.assertEqual((6, 180, 360), values.shape)
        self.assertAlmostEqual(0.74, values[0, 0, 0])
        self.assertAlmostEqual(0.74, values[-1, -1, -1])

    def test_from_two_inputs(self):
        calls = []

        def my_cube_func(analysed_sst: np.ndarray,
                         analysis_error: np.ndarray,
                         input_params: Dict[str, Any] = None,
                         dim_coords: Dict[str, np.ndarray] = None,
                         dim_ranges: Dict[str, Tuple[int, int]] = None) -> CubeFuncOutput:
            nonlocal calls
            calls.append((analysed_sst, analysis_error, input_params, dim_coords, dim_ranges))
            return analysed_sst.data + input_params['factor'] * analysis_error

        output_cube = compute_cube(my_cube_func,
                                   self.cube,
                                   input_var_names=['analysed_sst', 'analysis_error'],
                                   input_params=dict(factor=0.5))

        self.assertIsInstance(output_cube, xr.Dataset)
        self.assertIn('output', output_cube.data_vars)
        output_var = output_cube.output
        self.assertEqual(0, len(calls))
        self.assertEqual(('time', 'lat', 'lon'), output_var.dims)
        self.assertEqual((6, 180, 360), output_var.shape)
        self.assertEqual(((3, 3), (90, 90), (90, 90, 90, 90)), output_var.chunks)

        values = output_var.values
        self.assertEqual(2 * 2 * 4, len(calls))
        self.assertEqual((6, 180, 360), values.shape)
        self.assertAlmostEqual(275.3 + 0.5 * 2.1, values[0, 0, 0])
        self.assertAlmostEqual(275.3 + 0.5 * 2.1, values[-1, -1, -1])

    def test_from_two_inputs_reduce_time(self):
        calls = []

        def my_cube_func(analysed_sst: np.ndarray,
                         analysis_error: np.ndarray) -> CubeFuncOutput:
            nonlocal calls
            calls.append((analysed_sst, analysis_error))
            analysed_sst_mean = np.nanmean(analysed_sst, -1)
            analysis_error_max = np.nanmax(analysis_error, -1)
            return analysed_sst_mean + analysis_error_max

        output_dataset = compute_dataset(my_cube_func,
                                         self.cube,
                                         input_var_names=['analysed_sst', 'analysis_error'],
                                         output_var_name='analysed_sst_max',
                                         output_var_dims={'lat', 'lon'})

        self.assertIsInstance(output_dataset, xr.Dataset)
        self.assertIn('analysed_sst_max', output_dataset.data_vars)
        output_var = output_dataset.analysed_sst_max
        self.assertEqual(0, len(calls))  # call from dask with 0-dim chunks
        self.assertEqual(('lat', 'lon'), output_var.dims)
        self.assertEqual((180, 360), output_var.shape)
        self.assertEqual(((90, 90), (90, 90, 90, 90)), output_var.chunks)

        values = output_var.values
        self.assertEqual(2 * 4, len(calls))
        self.assertEqual((180, 360), values.shape)
        self.assertAlmostEqual(275.3 + 2.1, values[0, 0])
        self.assertAlmostEqual(275.3 + 2.1, values[-1, -1])

    def test_invalid_cube_func(self):
        def my_cube_func(analysis_error: np.ndarray) -> CubeFuncOutput:
            return analysis_error + 1

        with self.assertRaises(ValueError) as cm:
            compute_cube(my_cube_func,
                         self.cube,
                         input_var_names=['analysed_sst', 'analysis_error'],
                         input_params=dict(factor=0.5))
        self.assertEqual("invalid cube_func 'my_cube_func': expected 2 arguments, "
                         "but got analysis_error",
                         f'{cm.exception}')

        def my_cube_func(analysed_sst, dim_coords, analysis_error) -> CubeFuncOutput:
            return analysed_sst + analysis_error

        with self.assertRaises(ValueError) as cm:
            compute_cube(my_cube_func,
                         self.cube,
                         input_var_names=['analysed_sst', 'analysis_error'],
                         input_params=dict(factor=0.5))
        self.assertEqual("invalid cube_func 'my_cube_func': "
                         "any argument must occur before any of input_params, dim_coords, dim_ranges, "
                         "but got analysed_sst, dim_coords, analysis_error",
                         f'{cm.exception}')

        def my_cube_func(dim_ranges, *vars) -> CubeFuncOutput:
            return vars[0] + vars[0]

        with self.assertRaises(ValueError) as cm:
            compute_cube(my_cube_func,
                         self.cube,
                         input_var_names=['analysed_sst', 'analysis_error'],
                         input_params=dict(factor=0.5))
        self.assertEqual("invalid cube_func 'my_cube_func': "
                         "any argument must occur before any of input_params, dim_coords, dim_ranges, "
                         "but got dim_ranges before *vars",
                         f'{cm.exception}')

    def test_inspect(self):
        """This test is about making sure we get expected results from inspect.getfullargspec(func)"""

        def func():
            pass

        self._assert_inspect(func, exp_args=[], exp_kwonlyargs=[], exp_annotations={})

        # noinspection PyUnusedLocal
        def func(*input_vars):
            pass

        self._assert_inspect(func,
                             exp_args=[],
                             exp_varargs='input_vars',
                             exp_kwonlyargs=[],
                             exp_annotations={})

        # noinspection PyUnusedLocal
        def func(*input_vars, input_params, dim_coords):
            pass

        self._assert_inspect(func,
                             exp_args=[],
                             exp_varargs='input_vars',
                             exp_kwonlyargs=['input_params', 'dim_coords'],
                             exp_annotations={})

        # noinspection PyUnusedLocal
        def func(a, b, c, *input_vars, dim_ranges, input_params):
            pass

        self._assert_inspect(func,
                             exp_args=['a', 'b', 'c'],
                             exp_varargs='input_vars',
                             exp_kwonlyargs=['dim_ranges', 'input_params'],
                             exp_annotations={})

        # noinspection PyUnusedLocal
        def func(a, b, c, dim_ranges, input_params):
            pass

        self._assert_inspect(func,
                             exp_args=['a', 'b', 'c', 'dim_ranges', 'input_params'],
                             exp_kwonlyargs=[],
                             exp_annotations={})

    def _assert_inspect(self,
                        func,
                        exp_args=None,
                        exp_varargs=None,
                        exp_varkw=None,
                        exp_defaults=None,
                        exp_kwonlyargs=None,
                        exp_kwonlydefaults=None,
                        exp_annotations=None):
        import inspect
        argspec = inspect.getfullargspec(func)
        self.assertEqual(exp_args, argspec[0], msg='args')
        self.assertEqual(exp_varargs, argspec[1], msg='varargs')
        self.assertEqual(exp_varkw, argspec[2], msg='varkw')
        self.assertEqual(exp_defaults, argspec[3], msg='defaults')
        self.assertEqual(exp_kwonlyargs, argspec[4], msg='kwonlyargs')
        self.assertEqual(exp_kwonlydefaults, argspec[5], msg='kwonlydefaults')
        self.assertEqual(exp_annotations, argspec[6], msg='annotations')

    def test_xarray_apply_with_dim_reduction(self):
        def compute_block(block: np.ndarray, axis: int = None):
            # print('--> block:', block.shape)
            result = np.nanmean(block, axis=axis)
            # print('<-- result:', result.shape)
            return result

        def compute(obj: xr.Dataset, dim: str):
            return xr.apply_ufunc(compute_block, obj,
                                  kwargs=dict(axis=-1),  # note: apply always moves core dimensions to the end
                                  dask='parallelized',
                                  input_core_dims=[[dim]],
                                  output_dtypes=[np.float64])

        ds = xr.open_zarr(os.path.join(os.path.dirname(__file__), '../../examples/serve/demo/cube-1-250-250.zarr'))
        var = ds.conc_chl

        var_rechunked = var.chunk({'time': -1, 'lat': 250, 'lon': 250})
        var_computed = compute(var_rechunked, 'time')

        # This assertion succeeds
        self.assertEqual((1000, 2000), var_computed.shape)

        # Trigger computations of all chunks
        values = var_computed.values
        # This assertion succeeds fails, because values.shape is now (250, 250).
        # This must be an error in xarray or dask.
        self.assertEqual((1000, 2000), values.shape)

    def test_xarray_map_blocks(self):
        def compute_block(block: xr.Dataset):
            # print('--> block:', block)
            return block

        def compute(obj: xr.Dataset, dim: str):
            return obj.map_blocks(compute_block)

        ds = xr.open_zarr(os.path.join(os.path.dirname(__file__), '../../examples/serve/demo/cube-1-250-250.zarr'))
        var = ds.conc_chl

        var_computed = compute(var, 'time')

        # This assertion succeeds
        self.assertEqual((5, 1000, 2000), var_computed.shape)

        # Trigger computations of all chunks
        values = var_computed.values
        # This assertion succeeds fails, because values.shape is now (250, 250).
        # This must be an error in xarray or dask.
        self.assertEqual((5, 1000, 2000), values.shape)


# test/core/test_dsio.py

import os
import os.path
import unittest
from typing import Set

import boto3
import fsspec
import numpy as np
import pandas as pd
import xarray as xr

from test.s3test import MOTO_SERVER_ENDPOINT_URL, S3Test
from test.sampledata import new_test_dataset
from xcube.core.dsio import CsvDatasetIO, DatasetIO, MemDatasetIO, Netcdf4DatasetIO, ZarrDatasetIO, find_dataset_io, \
    query_dataset_io, get_path_or_s3_store, write_cube, split_s3_url, parse_s3_url_and_kwargs, open_cube
from xcube.core.dsio import open_dataset, write_dataset
from xcube.core.new import new_cube

TEST_NC_FILE = "test.nc"
TEST_NC_FILE_2 = "test-2.nc"


@unittest.skip('Execute manually to test with cube on S3, requires S3 credentials')
class OpenDatasetTest(unittest.TestCase):
    CUBE = 'eurodatacube-test/xcube-gen-f8207f0e-636d-4267-8ac5-abbdc5d121fc.zarr'

    def test_open_dataset(self):
        ds = open_dataset(f's3://{self.CUBE}')
        self.assertIn('B02', ds)
        print(ds)

    def test_plain(self):
        import s3fs

        def _open_cube(path):
            return xr.open_zarr(s3fs.S3Map(path, s3=s3fs.S3FileSystem()))

        ds = _open_cube(self.CUBE)
        self.assertIn('B02', ds)
        print(ds)


class OpenWriteDatasetTest(unittest.TestCase):
    def setUp(self):
        super().setUp()
        self.dataset = new_cube(variables=dict(precipitation=0.2, temperature=279.1))
        self.dataset.to_netcdf(TEST_NC_FILE, mode="w")
        self.dataset.close()

    def tearDown(self):
        self.dataset = None
        os.remove(TEST_NC_FILE)
        super().tearDown()

    def test_open_dataset(self):
        with open_dataset(TEST_NC_FILE) as ds:
            self.assertIsNotNone(ds)
            np.testing.assert_array_equal(ds.time.values, self.dataset.time.values)
            np.testing.assert_array_equal(ds.lat.values, self.dataset.lat.values)
            np.testing.assert_array_equal(ds.lon.values, self.dataset.lon.values)

    def test_write_dataset(self):

        dataset = new_cube()
        try:
            write_dataset(dataset, TEST_NC_FILE_2)
            self.assertTrue(os.path.isfile(TEST_NC_FILE_2))
        finally:
            if os.path.isfile(TEST_NC_FILE_2):
                os.remove(TEST_NC_FILE_2)


# noinspection PyAbstractClass
class MyDatasetIO(DatasetIO):

    def __init__(self):
        super().__init__("test")

    @property
    def description(self) -> str:
        return "test"

    @property
    def ext(self) -> str:
        return "test"

    @property
    def modes(self) -> Set[str]:
        return set()

    def fitness(self, input_path: str, path_type: str = None) -> float:
        return 0.0


class DatasetIOTest(unittest.TestCase):

    def test_read_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            ds_io.read('test.nc')

    def test_write_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.write(None, 'test.nc')

    def test_append_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.append(None, 'test.nc')

    def test_insert_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.insert(None, 0, 'test.nc')

    def test_replace_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.replace(None, 0, 'test.nc')

    def test_update_raises(self):
        ds_io = MyDatasetIO()
        with self.assertRaises(NotImplementedError):
            # noinspection PyTypeChecker
            ds_io.update(None, 'test.nc')


class MemDatasetIOTest(unittest.TestCase):

    def test_props(self):
        ds_io = MemDatasetIO()
        self.assertEqual('mem', ds_io.name)
        self.assertEqual('mem', ds_io.ext)
        self.assertEqual('In-memory dataset I/O', ds_io.description)
        self.assertEqual({'r', 'w', 'a'}, ds_io.modes)

    def test_fitness(self):
        ds_io = MemDatasetIO()

        self.assertEqual(0.75, ds_io.fitness("test.mem", path_type=None))
        self.assertEqual(0.75, ds_io.fitness("test.mem", path_type="file"))
        self.assertEqual(0.75, ds_io.fitness("test.mem", path_type="dir"))
        self.assertEqual(0.75, ds_io.fitness("http://dsio/test.mem", path_type="url"))

        self.assertEqual(0.0, ds_io.fitness("test.png", path_type=None))
        self.assertEqual(0.0, ds_io.fitness("test.png", path_type="file"))
        self.assertEqual(0.0, ds_io.fitness("test.png", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("http://dsio/test.png", path_type="url"))

        ds_io.write(xr.Dataset(), "bibo.odod")
        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type=None))
        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type="file"))
        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type="dir"))
        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type="url"))

    def test_read(self):
        ds_io = MemDatasetIO()
        with self.assertRaises(FileNotFoundError):
            ds_io.read('test.nc')
        ds1 = xr.Dataset()
        ds_io._datasets['test.nc'] = ds1
        ds2 = ds_io.read('test.nc')
        self.assertIs(ds2, ds1)

    def test_write(self):
        ds_io = MemDatasetIO()
        ds1 = xr.Dataset()
        ds_io.write(ds1, 'test.nc')
        ds2 = ds_io._datasets['test.nc']
        self.assertIs(ds2, ds1)

    def test_append(self):
        ds_io = MemDatasetIO()
        ds1 = new_test_dataset('2017-02-01', 180, temperature=1.2, precipitation=2.1)
        ds2 = new_test_dataset('2017-02-02', 180, temperature=2.3, precipitation=3.2)
        ds3 = new_test_dataset('2017-02-03', 180, temperature=3.4, precipitation=4.3)
        ds_io.append(ds1, 'test.nc')
        ds_io.append(ds2, 'test.nc')
        ds_io.append(ds3, 'test.nc')
        ds4 = ds_io._datasets.get('test.nc')
        self.assertIsNotNone(ds4)
        self.assertIn('time', ds4)
        self.assertIn('temperature', ds4)
        self.assertEqual(('time', 'lat', 'lon'), ds4.temperature.dims)
        self.assertEqual((3, 180, 360), ds4.temperature.shape)
        expected_time = xr.DataArray(pd.to_datetime(['2017-02-01', '2017-02-02', '2017-02-03']))
        np.testing.assert_equal(expected_time.values, ds4.time.values)


class Netcdf4DatasetIOTest(unittest.TestCase):

    def test_props(self):
        ds_io = Netcdf4DatasetIO()
        self.assertEqual('netcdf4', ds_io.name)
        self.assertEqual('nc', ds_io.ext)
        self.assertEqual('NetCDF-4 file format', ds_io.description)
        self.assertEqual({'a', 'r', 'w'}, ds_io.modes)

    def test_fitness(self):
        ds_io = Netcdf4DatasetIO()

        self.assertEqual(0.875, ds_io.fitness("test.nc", path_type=None))
        self.assertEqual(0.875, ds_io.fitness("test.hdf", path_type=None))
        self.assertEqual(0.875, ds_io.fitness("test.h5", path_type=None))

        self.assertEqual(1.0, ds_io.fitness("test.nc", path_type="file"))
        self.assertEqual(1.0, ds_io.fitness("test.hdf", path_type="file"))
        self.assertEqual(1.0, ds_io.fitness("test.h5", path_type="file"))

        self.assertEqual(0.0, ds_io.fitness("test.nc", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("test.hdf", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("test.h5", path_type="dir"))

        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.nc", path_type="url"))
        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.hdf", path_type="url"))
        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.h5", path_type="url"))

        self.assertEqual(0.125, ds_io.fitness("test.tif", path_type=None))
        self.assertEqual(0.25, ds_io.fitness("test.tif", path_type="file"))
        self.assertEqual(0.0, ds_io.fitness("test.tif", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.tif", path_type="url"))

    def test_read(self):
        ds_io = Netcdf4DatasetIO()
        with self.assertRaises(FileNotFoundError):
            ds_io.read('test.nc')


class ZarrDatasetIOTest(unittest.TestCase):

    def test_props(self):
        ds_io = ZarrDatasetIO()
        self.assertEqual('zarr', ds_io.name)
        self.assertEqual('zarr', ds_io.ext)
        self.assertEqual('Zarr file format (http://zarr.readthedocs.io)', ds_io.description)
        self.assertEqual({'a', 'r', 'w'}, ds_io.modes)

    def test_fitness(self):
        ds_io = ZarrDatasetIO()

        self.assertEqual(0.875, ds_io.fitness("test.zarr", path_type=None))
        self.assertEqual(0.0, ds_io.fitness("test.zarr", path_type="file"))
        self.assertEqual(1.0, ds_io.fitness("test.zarr", path_type="dir"))
        self.assertEqual(0.875, ds_io.fitness("http://dsio/test.zarr", path_type="url"))

        self.assertEqual(0.875, ds_io.fitness("test.zarr.zip", path_type=None))
        self.assertEqual(1.0, ds_io.fitness("test.zarr.zip", path_type="file"))
        self.assertEqual(0.0, ds_io.fitness("test.zarr.zip", path_type="dir"))
        self.assertEqual(0.0, ds_io.fitness("http://dsio/test.zarr.zip", path_type="url"))

        self.assertEqual(0.0, ds_io.fitness("test.png", path_type=None))
        self.assertEqual(0.0, ds_io.fitness("test.png", path_type="file"))
        self.assertEqual(0.25, ds_io.fitness("test.png", path_type="dir"))
        self.assertEqual(0.125, ds_io.fitness("http://dsio/test.png", path_type="url"))

    def test_read(self):
        ds_io = ZarrDatasetIO()
        with self.assertRaises(ValueError):
            ds_io.read('test.zarr')


class ZarrDatasetS3IOTest(S3Test):

    def test_write_to_and_read_from_s3(self):
        s3_conn = boto3.client('s3', endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        s3_conn.create_bucket(Bucket='upload_bucket', ACL='public-read')

        s3_kwargs = dict(key='test_fake_id', secret='test_fake_secret')
        s3_client_kwargs = {'endpoint_url': MOTO_SERVER_ENDPOINT_URL}

        ds1 = new_cube(width=36, height=18, variables=dict(chl=0.5, tsm=0.2))

        write_cube(ds1,
                   'upload_bucket/cube-1-250-250.zarr',
                   format_name='zarr',
                   s3_kwargs=s3_kwargs,
                   s3_client_kwargs=s3_client_kwargs)

        ds2 = open_cube('upload_bucket/cube-1-250-250.zarr',
                        format_name='zarr',
                        s3_kwargs=s3_kwargs,
                        s3_client_kwargs=s3_client_kwargs)

        self.assertEqual(set(ds1.coords), set(ds2.coords))
        self.assertEqual(set(ds1.data_vars), set(ds2.data_vars))


class CsvDatasetIOTest(unittest.TestCase):

    def test_props(self):
        ds_io = CsvDatasetIO()
        self.assertEqual('csv', ds_io.name)
        self.assertEqual('csv', ds_io.ext)
        self.assertEqual('CSV file format', ds_io.description)
        self.assertEqual({'r', 'w'}, ds_io.modes)

    def test_fitness(self):
        ds_io = CsvDatasetIO()
        self.assertEqual(0.875, ds_io.fitness("test.csv", path_type=None))
        self.assertEqual(1.0, ds_io.fitness("test.csv", path_type="file"))
        self.assertEqual(0.0, ds_io.fitness("test.csv", path_type="dir"))
        self.assertEqual(0.875, ds_io.fitness("http://dsio/test.csv", path_type="url"))

    def test_read(self):
        ds_io = CsvDatasetIO()
        with self.assertRaises(FileNotFoundError):
            ds_io.read('test.csv')


class FindDatasetIOTest(unittest.TestCase):

    def test_find_by_name(self):
        ds_io = find_dataset_io('netcdf4')
        self.assertIsInstance(ds_io, Netcdf4DatasetIO)

        ds_io = find_dataset_io('zarr', modes=['a'])
        self.assertIsInstance(ds_io, ZarrDatasetIO)
        ds_io = find_dataset_io('zarr', modes=['w'])
        self.assertIsInstance(ds_io, ZarrDatasetIO)
        ds_io = find_dataset_io('zarr', modes=['r'])
        self.assertIsInstance(ds_io, ZarrDatasetIO)

        ds_io = find_dataset_io('mem')
        self.assertIsInstance(ds_io, MemDatasetIO)

        ds_io = find_dataset_io('bibo', default=MemDatasetIO())
        self.assertIsInstance(ds_io, MemDatasetIO)

    def test_find_by_ext(self):
        ds_io = find_dataset_io('nc')
        self.assertIsInstance(ds_io, Netcdf4DatasetIO)


class QueryDatasetIOsTest(unittest.TestCase):
    def test_query_dataset_io(self):
        ds_ios = query_dataset_io()
        self.assertEqual(4, len(ds_ios))

    def test_query_dataset_io_with_fn(self):
        ds_ios = query_dataset_io(lambda ds_io: ds_io.name == 'mem')
        self.assertEqual(1, len(ds_ios))
        self.assertIsInstance(ds_ios[0], MemDatasetIO)


class ContextManagerTest(unittest.TestCase):
    def test_it(self):
        class A:
            def __init__(self):
                self.closed = False

            def __enter__(self):
                return self

            def __exit__(self, exception_type, exception_value, traceback):
                self.close()

            def close(self):
                self.closed = True

        def open_a():
            return A()

        with open_a() as a:
            pass

        self.assertEqual(True, a.closed)

        a = open_a()
        self.assertIsInstance(a, A)
        self.assertEqual(False, a.closed)


class GetPathOrObsStoreTest(S3Test):
    def test_path_or_store_read_from_bucket(self):
        s3_store, consolidated = get_path_or_s3_store(
            f'{MOTO_SERVER_ENDPOINT_URL}/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', mode='r')
        self.assertIsInstance(s3_store, fsspec.mapping.FSMap)
        self.assertEqual(False, consolidated)

    def test_path_or_store_write_to_bucket(self):
        s3_conn = boto3.client('s3', endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        s3_conn.create_bucket(Bucket='xcube-examples', ACL='public-read')

        s3_store, consolidated = get_path_or_s3_store(f'{MOTO_SERVER_ENDPOINT_URL}/xcube-examples/fake_cube.zarr',
                                                      s3_client_kwargs={'aws_access_key_id': 'some_fake_id',
                                                                        'aws_secret_access_key': 'some_fake_key'},
                                                      mode='w')
        self.assertIsInstance(s3_store, fsspec.mapping.FSMap)
        self.assertEqual(False, consolidated)

    def test_path_or_store_read_from_local(self):
        path, consolidated = get_path_or_s3_store('../examples/serve/demo/cube-1-250-250.zarr', mode='r')
        self.assertEqual('../examples/serve/demo/cube-1-250-250.zarr', path)
        self.assertEqual(False, consolidated)


class ParseObsUrlAndKwargsTest(unittest.TestCase):
    def test_http(self):
        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs(
            'https://s3.eu-central-1.amazonaws.com/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        self.assertEqual({'key': None, 'secret': None}, s3_kwargs)
        self.assertEqual({'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'}, s3_client_kwargs)

    def test_https_credentials(self):
        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs(
            'https://s3.eu-central-1.amazonaws.com/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', s3_client_kwargs={
                'provider_access_key_id': 'bibo',
                'provider_secret_access_key': '8625345',
            })
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        self.assertEqual({'key': 'bibo', 'secret': '8625345'}, s3_kwargs)
        self.assertEqual({'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'}, s3_client_kwargs)

        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs(
            'https://s3.eu-central-1.amazonaws.com/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', s3_client_kwargs={
                'aws_access_key_id': 'bibo',
                'aws_secret_access_key': '8625345',
            })
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        self.assertEqual({'key': 'bibo', 'secret': '8625345'}, s3_kwargs)
        self.assertEqual({'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'}, s3_client_kwargs)

    def test_s3_schema(self):
        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs('s3://xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                                                                    s3_kwargs={'anon': True},
                                                                    s3_client_kwargs={})
        self.assertEqual('s3://xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        self.assertEqual({'anon': True, 'key': None, 'secret': None}, s3_kwargs)
        self.assertEqual({}, s3_client_kwargs)

    def test_endpoint_url(self):
        root, s3_kwargs, s3_client_kwargs = parse_s3_url_and_kwargs(
            'eurodatacube-test/demo/xcube-gen-489bb051.zarr',
            s3_kwargs={'anon': True},
            s3_client_kwargs={
                'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'
            }
        )
        self.assertEqual('eurodatacube-test/demo/xcube-gen-489bb051.zarr', root)
        self.assertEqual({'anon': True, 'key': None, 'secret': None}, s3_kwargs)
        self.assertEqual({'endpoint_url': 'https://s3.eu-central-1.amazonaws.com'}, s3_client_kwargs)


class SplitBucketUrlTest(unittest.TestCase):
    def test_http(self):
        endpoint_url, root = split_s3_url(
            'https://s3.eu-central-1.amazonaws.com/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual('https://s3.eu-central-1.amazonaws.com', endpoint_url)
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)
        endpoint_url, root = split_s3_url('https://xcube-serve:9098/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual('https://xcube-serve:9098', endpoint_url)
        self.assertEqual('xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)

    def test_s3(self):
        endpoint_url, root = split_s3_url('s3://xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual(None, endpoint_url)
        self.assertEqual('s3://xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)

    def test_local(self):
        endpoint_url, root = split_s3_url('/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual(None, endpoint_url)
        self.assertEqual('/xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', root)


# test/core/test_dump.py

import unittest

from test.sampledata import new_test_dataset
from xcube.core.dump import dump_dataset


class DumpDatasetTest(unittest.TestCase):
    def test_dump_dataset(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05"],
                                   precipitation=0.4, temperature=275.2)
        for var in dataset.variables.values():
            var.encoding.update({"_FillValue": 999.0})

        print(dataset.dims)

        text = dump_dataset(dataset)
        self.assertIn("<xarray.Dataset>", text)
        self.assertIn("Dimensions:        (time: 5, lat: 180, lon: 360)\n", text)
        self.assertIn("Coordinates:\n", text)
        self.assertIn("  * lon            (lon) float64 ", text)
        self.assertIn("Data variables:\n", text)
        self.assertIn("    precipitation  (time, lat, lon) float64 ", text)
        self.assertNotIn("Encoding for coordinate variable 'lat':\n", text)
        self.assertNotIn("Encoding for data variable 'temperature':\n", text)
        self.assertNotIn("    _FillValue:  999.0\n", text)

        text = dump_dataset(dataset, show_var_encoding=True)
        self.assertIn("<xarray.Dataset>", text)
        self.assertIn("Dimensions:        (time: 5, lat: 180, lon: 360)\n", text)
        self.assertIn("Coordinates:\n", text)
        self.assertIn("  * lon            (lon) float64 ", text)
        self.assertIn("Data variables:\n", text)
        self.assertIn("    precipitation  (time, lat, lon) float64 ", text)
        self.assertIn("Encoding for coordinate variable 'lat':\n", text)
        self.assertIn("Encoding for data variable 'temperature':\n", text)
        self.assertIn("    _FillValue:  999.0\n", text)

        text = dump_dataset(dataset, ["precipitation"])
        self.assertIn("<xarray.DataArray 'precipitation' (time: 5, lat: 180, lon: 360)>\n", text)
        self.assertNotIn("Encoding:\n", text)
        self.assertNotIn("    _FillValue:  999.0", text)

        text = dump_dataset(dataset, ["precipitation"], show_var_encoding=True)
        self.assertIn("<xarray.DataArray 'precipitation' (time: 5, lat: 180, lon: 360)>\n", text)
        self.assertIn("Encoding:\n", text)
        self.assertIn("    _FillValue:  999.0", text)


# test/core/test_edit.py

import unittest

import yaml
import zarr

from test.sampledata import create_highroc_dataset
from xcube.core.new import new_cube
from xcube.core.chunk import chunk_dataset
from xcube.constants import FORMAT_NAME_ZARR
from xcube.core.dsio import rimraf
from xcube.core.edit import edit_metadata
from xcube.core.optimize import optimize_dataset

TEST_CUBE = create_highroc_dataset()

TEST_CUBE_COORDS = chunk_dataset(new_cube(time_periods=3, variables=dict(A=0.5, B=-1.5)),
                                 chunk_sizes=dict(time=1, lat=180, lon=360), format_name=FORMAT_NAME_ZARR)

TEST_CUBE_ZARR = 'test.zarr'

TEST_CUBE_ZARR_COORDS = 'test_coords.zarr'

TEST_CUBE_ZARR_OPTIMIZED = 'test-optimized.zarr'

TEST_CUBE_ZARR_EDIT = 'test_edited_meta.zarr'

TEST_CUBE_ZARR_OPTIMIZED_EDIT = 'test_optimized_edited_meta.zarr'

TEST_NEW_META = {'output_metadata': {'creator_name': 'Brockmann Consult GmbH with love',
                                     'creator_url': 'www.some_very_nice_url.com',
                                     'geospatial_lat_max': 'something around the north pole.'},
                 'conc_chl': {'units': 'happiness'},
                 'some_crazy_var': {'units': 'happiness'}}

TEST_NEW_META_YML = 'test_new_meta.yml'


class EditVariablePropsTest(unittest.TestCase):

    def setUp(self):
        rimraf(TEST_CUBE_ZARR)
        rimraf(TEST_CUBE_ZARR_COORDS)
        rimraf(TEST_NEW_META_YML)
        rimraf(TEST_CUBE_ZARR_EDIT)
        rimraf(TEST_CUBE_ZARR_OPTIMIZED)
        rimraf(TEST_CUBE_ZARR_OPTIMIZED_EDIT)
        TEST_CUBE.to_zarr(TEST_CUBE_ZARR)
        TEST_CUBE_COORDS.to_zarr(TEST_CUBE_ZARR_COORDS)
        with open(TEST_NEW_META_YML, 'w') as outfile:
            yaml.dump(TEST_NEW_META, outfile, default_flow_style=False)

    def tearDown(self):
        rimraf(TEST_CUBE_ZARR)
        rimraf(TEST_CUBE_ZARR_COORDS)
        rimraf(TEST_NEW_META_YML)
        rimraf(TEST_CUBE_ZARR_EDIT)
        rimraf(TEST_CUBE_ZARR_OPTIMIZED)
        rimraf(TEST_CUBE_ZARR_OPTIMIZED_EDIT)

    def test_edit_metadata(self):
        edit_metadata(TEST_CUBE_ZARR, metadata_path=TEST_NEW_META_YML, in_place=False,
                      output_path=TEST_CUBE_ZARR_EDIT, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertNotIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_name', ds2.attrs.keys())

    def test_update_coords_metadata(self):
        edit_metadata(TEST_CUBE_ZARR, metadata_path=TEST_NEW_META_YML, update_coords=True, in_place=False,
                      output_path=TEST_CUBE_ZARR_EDIT, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertIn('geospatial_lon_units', ds2.attrs.keys())
        self.assertEqual('degrees_east', ds2.attrs.__getitem__('geospatial_lon_units'))
        self.assertNotIn('geospatial_lat_max', ds1.attrs.keys())
        self.assertNotIn('geospatial_lat_max', ds2.attrs.keys())

    def test_update_coords_metadata_only(self):
        ds1 = zarr.open(TEST_CUBE_ZARR_COORDS)
        delete_list = ['geospatial_lat_max', 'geospatial_lat_min', 'geospatial_lat_units', 'geospatial_lon_max',
                       'geospatial_lon_min', 'geospatial_lon_units', 'time_coverage_end', 'time_coverage_start']
        for attr in ds1.attrs.keys():
            if attr in delete_list:
                ds1.attrs.__delitem__(attr)
        edit_metadata(TEST_CUBE_ZARR_COORDS, update_coords=True, in_place=False,
                      output_path=TEST_CUBE_ZARR_EDIT, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR_COORDS)
        ds2 = zarr.open(TEST_CUBE_ZARR_EDIT)
        for attr in delete_list:
            self.assertNotIn(attr, ds1.attrs.keys())
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertIn('geospatial_lat_max', ds2.attrs.keys())
        self.assertIn('geospatial_lat_min', ds2.attrs.keys())
        self.assertIn('geospatial_lat_units', ds2.attrs.keys())
        self.assertIn('geospatial_lon_max', ds2.attrs.keys())
        self.assertEqual(180.0, ds2.attrs.__getitem__('geospatial_lon_max'))
        self.assertEqual(-180.0, ds2.attrs.__getitem__('geospatial_lon_min'))
        self.assertEqual('2010-01-04T00:00:00', ds2.attrs.__getitem__('time_coverage_end'))
        self.assertEqual('2010-01-01T00:00:00', ds2.attrs.__getitem__('time_coverage_start'))
        self.assertEqual('degrees_east', ds2.attrs.__getitem__('geospatial_lon_units'))

    def test_edit_metadata_in_place(self):
        edit_metadata(TEST_CUBE_ZARR, metadata_path=TEST_NEW_META_YML, in_place=True, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR)
        self.assertEqual(36, ds1.__len__())
        self.assertEqual('14-APR-2017 10:27:50.183264', ds1.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds1['conc_chl'].attrs.__getitem__('units'))
        self.assertIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_url', ds1.attrs.keys())
        self.assertEqual('Brockmann Consult GmbH with love', ds1.attrs.__getitem__('creator_name'))
        self.assertEqual('www.some_very_nice_url.com', ds1.attrs.__getitem__('creator_url'))

    def test_edit_zmetadata(self):
        optimize_dataset(TEST_CUBE_ZARR, unchunk_coords=True, output_path=TEST_CUBE_ZARR_OPTIMIZED)
        edit_metadata(TEST_CUBE_ZARR_OPTIMIZED, metadata_path=TEST_NEW_META_YML, in_place=False,
                      output_path=TEST_CUBE_ZARR_OPTIMIZED_EDIT, monitor=print)
        ds1 = zarr.open(TEST_CUBE_ZARR)
        ds2 = zarr.convenience.open_consolidated(TEST_CUBE_ZARR_OPTIMIZED_EDIT)
        self.assertEqual(ds1.__len__(), ds2.__len__())
        self.assertEqual(ds1.attrs.__getitem__('start_date'), ds2.attrs.__getitem__('start_date'))
        self.assertEqual('happiness', ds2['conc_chl'].attrs.__getitem__('units'))
        self.assertNotIn('creator_name', ds1.attrs.keys())
        self.assertIn('creator_name', ds2.attrs.keys())

    def test_failures(self):
        with self.assertRaises(RuntimeError) as cm:
            edit_metadata('pippo', in_place=True, exception_type=RuntimeError, metadata_path=TEST_NEW_META_YML)
        self.assertEqual('Input path must point to ZARR dataset directory.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            edit_metadata(TEST_CUBE_ZARR, exception_type=RuntimeError, metadata_path=TEST_NEW_META_YML)
        self.assertEqual('Output path must be given.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            edit_metadata(TEST_CUBE_ZARR, output_path=TEST_CUBE_ZARR, exception_type=RuntimeError,
                          metadata_path=TEST_NEW_META_YML)
        self.assertEqual('Output path already exists.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            edit_metadata(TEST_CUBE_ZARR, output_path='./' + TEST_CUBE_ZARR, exception_type=RuntimeError,
                          metadata_path=TEST_NEW_META_YML)
        self.assertEqual('Output path already exists.', f'{cm.exception}')


# test/core/test_evaluate.py

import unittest

import dask.array as da
import numpy as np
import xarray as xr
from xcube.core.evaluate import evaluate_dataset

nan = float('nan')


class EvaluateDatasetTest(unittest.TestCase):
    @classmethod
    def get_test_dataset(cls):
        return xr.Dataset(dict(a=(('y', 'x'), [[0.1, 0.2, 0.4, 0.1], [0.5, 0.1, 0.2, 0.3]]),
                               b=(('y', 'x'), [[0.4, 0.3, 0.2, 0.4], [0.1, 0.2, 0.5, 0.1]],
                                  dict(valid_pixel_expression='a >= 0.2')),
                               c=((), 1.0),
                               d=((), 1.0, dict(expression='a * b'))),
                          coords=dict(x=(('x',), [1, 2, 3, 4]), y=(('y',), [1, 2])),
                          attrs=dict(title='test_compute_dataset')).chunk(dict(x=2, y=2))

    def test_compute_dataset_without_processed_variables(self):
        dataset = self.get_test_dataset()
        computed_dataset = evaluate_dataset(dataset)
        self.assertIsNot(computed_dataset, dataset)
        self.assertIn('x', computed_dataset)
        self.assertIn('y', computed_dataset)
        self.assertIn('a', computed_dataset)
        self.assertIn('b', computed_dataset)
        self.assertIn('c', computed_dataset)
        self.assertIn('d', computed_dataset)
        self.assertIn('x', computed_dataset.coords)
        self.assertIn('y', computed_dataset.coords)
        self.assertIn('title', computed_dataset.attrs)
        self.assertEqual((2, 4), computed_dataset.a.shape)
        self.assertEqual((2, 4), computed_dataset.b.shape)
        self.assertEqual((), computed_dataset.c.shape)
        self.assertNotIn('expression', computed_dataset.c.attrs)
        self.assertEqual((2, 4), computed_dataset.d.shape)
        self.assertIn('expression', computed_dataset.d.attrs)
        np.testing.assert_array_almost_equal(computed_dataset.a.values,
                                             np.array([[0.1, 0.2, 0.4, 0.1], [0.5, 0.1, 0.2, 0.3]]))
        np.testing.assert_array_almost_equal(computed_dataset.b.values,
                                             np.array([[nan, 0.3, 0.2, nan], [0.1, nan, 0.5, 0.1]]))
        np.testing.assert_array_almost_equal(computed_dataset.c.values,
                                             np.array([1.]))
        np.testing.assert_array_almost_equal(computed_dataset.d.values,
                                             np.array([[0.04, 0.06, 0.08, 0.04],
                                                       [0.05, 0.02, 0.1, 0.03]]))

    def test_compute_dataset_with_processed_variables(self):
        dataset = self.get_test_dataset()
        computed_dataset = evaluate_dataset(dataset,
                                            processed_variables=[('a', None),
                                                                 ('b', dict(valid_pixel_expression=None)),
                                                                 ('c', dict(expression='a + b',
                                                                            load=True)),
                                                                 ('d', dict(valid_pixel_expression='c > 0.4',
                                                                            load=True))])
        self.assertIsNot(computed_dataset, dataset)
        self.assertIn('x', computed_dataset)
        self.assertIn('y', computed_dataset)
        self.assertIn('a', computed_dataset)
        self.assertIn('b', computed_dataset)
        self.assertIn('c', computed_dataset)
        self.assertIn('d', computed_dataset)
        self.assertIn('x', computed_dataset.coords)
        self.assertIn('y', computed_dataset.coords)
        self.assertIn('title', computed_dataset.attrs)
        self.assertEqual((2, 4), computed_dataset.a.shape)
        self.assertEqual((2, 4), computed_dataset.b.shape)
        self.assertEqual((2, 4), computed_dataset.c.shape)
        self.assertIsInstance(computed_dataset.a.data, da.Array)
        self.assertIsInstance(computed_dataset.b.data, da.Array)
        self.assertIsInstance(computed_dataset.c.data, np.ndarray)  # load=True --> load c as numpy array
        self.assertIsInstance(computed_dataset.d.data, np.ndarray)  # load=True --> load d as numpy array
        self.assertIn('expression', computed_dataset.c.attrs)
        self.assertEqual((2, 4), computed_dataset.d.shape)
        self.assertIn('expression', computed_dataset.d.attrs)
        np.testing.assert_array_almost_equal(computed_dataset.a.values,
                                             np.array([[0.1, 0.2, 0.4, 0.1], [0.5, 0.1, 0.2, 0.3]]))
        np.testing.assert_array_almost_equal(computed_dataset.b.values,
                                             np.array([[0.4, 0.3, 0.2, 0.4], [0.1, 0.2, 0.5, 0.1]]))
        np.testing.assert_array_almost_equal(computed_dataset.c.values,
                                             np.array([[0.5, 0.5, 0.6, 0.5], [0.6, 0.3, 0.7, 0.4]]))
        np.testing.assert_array_almost_equal(computed_dataset.d.values,
                                             np.array([[0.04, 0.06, 0.08, 0.04], [0.05, nan, 0.1, nan]]))


# test/core/test_extract.py

import unittest

import numpy as np
import pandas as pd
import xarray as xr

from xcube.core.extract import get_cube_point_indexes
from xcube.core.extract import get_cube_values_for_points
from xcube.core.extract import get_dataset_indexes
from xcube.core.new import new_cube


# noinspection PyMethodMayBeStatic
class GetCubeValuesForPointsTest(unittest.TestCase):
    def _new_test_cube(self):
        return new_cube(width=2000,
                        height=1000,
                        x_start=0.0,
                        y_start=50.0,
                        x_res=4.0 / 2000,
                        time_start="2010-01-01",
                        time_periods=20,
                        variables=dict(precipitation=0.6, temperature=276.2))

    def _new_test_points(self):
        return dict(time=np.array(["2010-01-04", "2009-04-09",
                                   "2010-01-08", "2010-01-06",
                                   "2010-01-20", "2010-01-01",
                                   "2010-01-05", "2010-01-03",
                                   ], dtype="datetime64[ns]"),
                    # Note "lat" is a list (see issue #434)
                    lat=[50.0, 51.3, 49.7, 50.1, 51.9, 50.8, 50.2, 52.0],
                    lon=np.array([0.0, 0.1, 0.4, 2.9, 1.6, 0.7, -0.5, 4.0]))

    def test_get_cube_point_values_from_data_frame(self):
        self._test_get_cube_point_values(pd.DataFrame(self._new_test_points()))

    def test_get_cube_point_values_from_dataset(self):
        self._test_get_cube_point_values(xr.Dataset(self._new_test_points()))

    def test_get_cube_point_values_from_dict(self):
        self._test_get_cube_point_values(self._new_test_points())

    def _test_get_cube_point_values(self, points):
        cube = self._new_test_cube()

        expected_prec_values = [0.6, np.nan, np.nan,
                                0.6, 0.6, 0.6, np.nan, 0.6]
        expected_temp_values = [276.2, np.nan, np.nan,
                                276.2, 276.2, 276.2, np.nan, 276.2]
        expected_time_index = [3, -1, 7, 5, 19, 0, 4, 2]
        expected_lat_index = [0, 650, -1, 50, 950, 400, 100, 999]
        expected_lon_index = [0, 50, 200, 1450, 800, 349, -1, 1999]
        values = get_cube_values_for_points(cube, points, include_indexes=True)
        self.assertIsInstance(values, xr.Dataset)
        self.assertEqual(5, len(values.data_vars))
        self.assertEqual({
            'precipitation',
            'temperature',
            'lat_index',
            'lon_index',
            'time_index'},
            set(values.data_vars))
        np.testing.assert_array_almost_equal(np.array(expected_prec_values,
                                                      dtype=np.float64),
                                             values["precipitation"].values)
        np.testing.assert_array_almost_equal(np.array(expected_temp_values,
                                                      dtype=np.float64),
                                             values["temperature"].values)
        np.testing.assert_array_equal(np.array(expected_time_index,
                                               dtype=np.int64),
                                      values["time_index"].values)
        np.testing.assert_array_equal(np.array(expected_lat_index,
                                               dtype=np.int64),
                                      values["lat_index"].values)
        np.testing.assert_array_equal(np.array(expected_lon_index,
                                               dtype=np.int64),
                                      values["lon_index"].values)

    def test_get_cube_point_indexes(self):
        cube = self._new_test_cube()

        expected_time_index = [3., np.nan, 7.,
                               5., 19., 0., 4., 2.]
        expected_lat_index = [0., 650., np.nan,
                              50., 950, 400., 100., 1000. - 1e-9]
        expected_lon_index = [0., 50., 200.,
                              1450., 800., 350., np.nan, 2000. - 1e-9]

        indexes = get_cube_point_indexes(cube, self._new_test_points())
        self.assertIsInstance(indexes, xr.Dataset)
        self.assertEqual(3, len(indexes.data_vars))
        self.assertEqual({'time_index', 'lat_index', 'lon_index'},
                         set(indexes.data_vars))
        np.testing.assert_array_almost_equal(np.array(expected_time_index,
                                                      dtype=np.float64),
                                             indexes["time_index"].values)
        np.testing.assert_array_almost_equal(np.array(expected_lat_index,
                                                      dtype=np.float64),
                                             indexes["lat_index"].values)
        np.testing.assert_array_almost_equal(np.array(expected_lon_index,
                                                      dtype=np.float64),
                                             indexes["lon_index"].values)


# noinspection PyMethodMayBeStatic
class GetDatasetIndexesTest(unittest.TestCase):

    def test_get_dataset_indexes_for_single_cell(self):
        dataset = new_cube(width=360, height=180, drop_bounds=True)
        cell = dataset.isel(time=2, lat=20, lon=30)
        with self.assertRaises(ValueError) as cm:
            get_dataset_indexes(cell, "lon", np.array([-149.5]))
        self.assertEqual("cannot determine cell boundaries"
                         " for coordinate variable 'lon' of size 1",
                         f"{cm.exception}")

    def test_get_dataset_indexes_with_bounds(self):
        dataset = new_cube(width=360, height=180,
                           drop_bounds=False)
        self._assert_get_dataset_indexes_works(dataset)

    def test_get_dataset_indexes_without_bounds(self):
        dataset = new_cube(width=360, height=180,
                           drop_bounds=True)
        self._assert_get_dataset_indexes_works(dataset)

    def test_get_dataset_indexes_with_bounds_inverse_lat(self):
        dataset = new_cube(width=360, height=180,
                           inverse_y=True, drop_bounds=False)
        self._assert_get_dataset_indexes_works(dataset, inverse_lat=True)

    def test_get_dataset_indexes_without_bounds_inverse_lat(self):
        dataset = new_cube(width=360, height=180,
                           inverse_y=True, drop_bounds=True)
        self._assert_get_dataset_indexes_works(dataset, inverse_lat=True)

    def _assert_get_dataset_indexes_works(self, dataset, inverse_lat=False):
        # - lon -------------------------------------

        lon_coords = np.array([-190, -180., -179,
                               -10.4, 0., 10.4, 179., 180.0, 190])
        expected_lon_int64 = np.array([-1, 0, 1,
                                       169, 180, 190,
                                       359, 359, -1],
                                      dtype=np.int64)
        expected_lon_float64 = np.array([np.nan, 0., 1.,
                                         169.6, 180., 190.4,
                                         359., 360., np.nan],
                                        dtype=np.float64)

        indexes = get_dataset_indexes(dataset, "lon", lon_coords,
                                      index_dtype=np.int32)
        np.testing.assert_array_equal(indexes, expected_lon_int64)
        indexes = get_dataset_indexes(dataset, "lon", lon_coords)
        np.testing.assert_array_almost_equal(indexes, expected_lon_float64)

        # - lat -------------------------------------

        lat_coords = np.array([-100, -90., -89,
                               -10.4, 0., 10.4, 89., 90.0, 100])
        if not inverse_lat:
            expected_lat_int64 = np.array([-1, 0, 1,
                                           79, 90, 100, 179, 179, -1],
                                          dtype=np.int64)
            expected_lat_float64 = np.array([np.nan, 0., 1., 79.6,
                                             90., 100.4, 179.,
                                             180., np.nan],
                                            dtype=np.float64)
        else:
            expected_lat_int64 = np.array([-1, 179, 179,
                                           100, 90, 79, 1, 0, -1],
                                          dtype=np.int64)
            expected_lat_float64 = np.array([np.nan, 180., 179.,
                                             100.4, 90., 79.6, 1., 0., np.nan],
                                            dtype=np.float64)

        indexes = get_dataset_indexes(dataset, "lat", lat_coords,
                                      index_dtype=np.int32)
        np.testing.assert_array_equal(indexes, expected_lat_int64)
        indexes = get_dataset_indexes(dataset, "lat", lat_coords)
        np.testing.assert_array_almost_equal(indexes, expected_lat_float64)

        # - time ------------------------------------

        time_coords = np.array(
            ["2010-01-03", "2010-01-02T23:15", "2010-05-15",
             "2010-01-01T12:00", "2010-01-05", "2009-12-07"],
            dtype="datetime64[ns]")
        expected_time_int64 = np.array([2, 1, -1, 0, 4, -1],
                                       dtype=np.int64)
        expected_time_float64 = np.array([2., 1.96875, np.nan,
                                          0.5, 4., np.nan],
                                         dtype=np.float64)

        indexes = get_dataset_indexes(dataset, "time", time_coords,
                                      index_dtype=np.int32)
        np.testing.assert_array_equal(indexes, expected_time_int64)
        indexes = get_dataset_indexes(dataset, "time", time_coords)
        np.testing.assert_array_almost_equal(indexes, expected_time_float64)


# test/core/test_geom.py

import unittest

import dask.array as da
import geopandas as gpd
import numpy as np
import shapely.geometry
import shapely.wkt
import xarray as xr

from xcube.core.chunk import chunk_dataset
from xcube.core.geom import clip_dataset_by_geometry
from xcube.core.geom import convert_geometry
from xcube.core.geom import get_dataset_bounds
from xcube.core.geom import get_dataset_geometry
from xcube.core.geom import get_geometry_mask
from xcube.core.geom import is_dataset_y_axis_inverted
from xcube.core.geom import is_lon_lat_dataset
from xcube.core.geom import mask_dataset_by_geometry
from xcube.core.geom import normalize_geometry
from xcube.core.geom import rasterize_features
from xcube.core.new import new_cube
from xcube.util.types import normalize_scalar_or_pair

nan = np.nan


class RasterizeFeaturesIntoDataset(unittest.TestCase):
    def test_rasterize_geo_data_frame_lonlat(self):
        self._test_rasterize_features(self.get_geo_data_frame_features(),
                                      'lon', 'lat')

    def test_rasterize_geo_data_frame_lonlat_chunked(self):
        self._test_rasterize_features(self.get_geo_data_frame_features(),
                                      'lon', 'lat', with_var=True)

    def test_rasterize_geo_data_frame_lonlat_fix_chunks(self):
        self._test_rasterize_features(self.get_geo_data_frame_features(),
                                      'lon', 'lat', tile_size=4)

    def test_rasterize_geo_data_frame_xy(self):
        self._test_rasterize_features(self.get_geo_data_frame_features(),
                                      'x', 'y')

    def test_rasterize_geo_data_frame_xy_chunked(self):
        self._test_rasterize_features(self.get_geo_data_frame_features(),
                                      'x', 'y', with_var=True)

    def test_rasterize_geo_data_frame_xy_fix_chunks(self):
        self._test_rasterize_features(self.get_geo_data_frame_features(),
                                      'x', 'y', tile_size=(3, 4))

    def test_rasterize_geo_json_lonlat(self):
        self._test_rasterize_features(self.get_geo_json_features(),
                                      'lon', 'lat')

    def test_rasterize_geo_json_lonlat_chunked(self):
        self._test_rasterize_features(self.get_geo_json_features(),
                                      'lon', 'lat', with_var=True)

    def test_rasterize_geo_json_xy(self):
        self._test_rasterize_features(self.get_geo_json_features(),
                                      'x', 'y')

    def test_rasterize_geo_json_xy_chunked(self):
        self._test_rasterize_features(self.get_geo_json_features(),
                                      'x', 'y', with_var=True)

    def test_rasterize_geo_json_xy_chunked_inverse_y(self):
        self._test_rasterize_features(self.get_geo_json_features(),
                                      'x', 'y',
                                      with_var=True,
                                      inverse_y=True)

    def test_rasterize_geo_json_xy_fixed_chunks_inverse_y(self):
        self._test_rasterize_features(self.get_geo_json_features(),
                                      'x', 'y',
                                      tile_size=4,
                                      inverse_y=True)

    def test_rasterize_invalid_feature(self):
        features = self.get_geo_json_features()
        with self.assertRaises(ValueError) as cm:
            rasterize_features(new_cube(),
                               features,
                               ['missing'])
        self.assertEqual("feature property 'missing' not found",
                         f'{cm.exception}')

    def _test_rasterize_features(self,
                                 features,
                                 x_name, y_name,
                                 with_var=False,
                                 tile_size=None,
                                 inverse_y=False):
        width = 10
        height = 10
        dataset = new_cube(
            width=width, height=height,
            x_name=x_name, y_name=y_name,
            x_res=10,
            x_start=-50, y_start=-50,
            variables=dict(d=12.5) if with_var else None,
            inverse_y=inverse_y
        )
        if with_var:
            dataset['d'] = dataset['d'].chunk({x_name: 4,
                                               y_name: 3,
                                               'time': 1})

        dataset = rasterize_features(dataset,
                                     features,
                                     ['a', 'b', 'c'],
                                     var_props=dict(
                                         b=dict(name='b', dtype=np.float32,
                                                fill_value=np.nan,
                                                attrs=dict(units='meters')),
                                         c=dict(name='c2', dtype=np.uint8,
                                                fill_value=0, converter=int)
                                     ),
                                     tile_size=tile_size,
                                     in_place=False)

        self.assertIsNotNone(dataset)
        if with_var:
            cy, cx = (3, 3, 3, 1), (4, 4, 2)
        elif tile_size:
            tw, th = normalize_scalar_or_pair(tile_size)
            cy, cx = da.core.normalize_chunks((th, tw), shape=(height, width))
        else:
            cy, cx = (10,), (10,)
        if not inverse_y:
            cy = tuple(reversed(cy))
        self.assertEqual({x_name: cx, y_name: cy}, dataset.chunks)

        self.assertIn(x_name, dataset.coords)
        self.assertIn(y_name, dataset.coords)
        self.assertIn('time', dataset.coords)
        self.assertIn('a', dataset)
        self.assertIn('b', dataset)
        self.assertIn('c2', dataset)
        self.assertEquals((10, 10), dataset.a.shape)
        self.assertEquals((10, 10), dataset.b.shape)
        self.assertEquals((10, 10), dataset.c2.shape)
        self.assertEquals(np.float64, dataset.a.dtype)
        self.assertEquals(np.float32, dataset.b.dtype)
        self.assertEquals(np.uint8, dataset.c2.dtype)
        self.assertEquals({}, dataset.a.attrs)
        self.assertEquals({'units': 'meters'}, dataset.b.attrs)
        self.assertEquals({}, dataset.c2.attrs)
        self.assertEquals((y_name, x_name), dataset.a.dims)
        self.assertEquals((y_name, x_name), dataset.b.dims)
        self.assertEquals((y_name, x_name), dataset.c2.dims)
        actual_a_values = dataset.a.values
        expected_a_values = np.array(
            [[0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8],
             [0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8],
             [0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8],
             [0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8],
             [0.6, 0.6, 0.6, 0.6, 0.6, nan, nan, 0.8, 0.8, 0.8],
             [0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7],
             [0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7],
             [0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7],
             [0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7],
             [0.5, 0.5, 0.5, 0.5, 0.5, nan, nan, 0.7, 0.7, 0.7]]
        )
        if inverse_y:
            expected_a_values = expected_a_values[::-1, :]
        np.testing.assert_almost_equal(expected_a_values,
                                       actual_a_values)
        actual_b_values = dataset.b.values
        expected_b_values = np.array(
            [[2.2, 2.2, 2.2, 2.2, 2.2, nan, nan, 2.4, 2.4, 2.4],
             [2.2, 2.2, 2.2, 2.2, 2.2, nan, nan, 2.4, 2.4, 2.4],
             [2.2, 2.2, 2.2, 2.2, 2.2, nan, nan, 2.4, 2.4, 2.4],
             [2.2, 2.2, 2.2, 2.2, 2.2, nan, nan, 2.4, 2.4, 2.4],
             [2.2, 2.2, 2.2, 2.2, 2.2, nan, nan, 2.4, 2.4, 2.4],
             [2.1, 2.1, 2.1, 2.1, 2.1, nan, nan, 2.3, 2.3, 2.3],
             [2.1, 2.1, 2.1, 2.1, 2.1, nan, nan, 2.3, 2.3, 2.3],
             [2.1, 2.1, 2.1, 2.1, 2.1, nan, nan, 2.3, 2.3, 2.3],
             [2.1, 2.1, 2.1, 2.1, 2.1, nan, nan, 2.3, 2.3, 2.3],
             [2.1, 2.1, 2.1, 2.1, 2.1, nan, nan, 2.3, 2.3, 2.3]]
        )
        if inverse_y:
            expected_b_values = expected_b_values[::-1, :]
        np.testing.assert_almost_equal(expected_b_values,
                                       actual_b_values)
        actual_c_values = dataset.c2.values
        expected_c_values = np.array(
            [[8, 8, 8, 8, 8, 0, 0, 6, 6, 6],
             [8, 8, 8, 8, 8, 0, 0, 6, 6, 6],
             [8, 8, 8, 8, 8, 0, 0, 6, 6, 6],
             [8, 8, 8, 8, 8, 0, 0, 6, 6, 6],
             [8, 8, 8, 8, 8, 0, 0, 6, 6, 6],
             [9, 9, 9, 9, 9, 0, 0, 7, 7, 7],
             [9, 9, 9, 9, 9, 0, 0, 7, 7, 7],
             [9, 9, 9, 9, 9, 0, 0, 7, 7, 7],
             [9, 9, 9, 9, 9, 0, 0, 7, 7, 7],
             [9, 9, 9, 9, 9, 0, 0, 7, 7, 7]],
            dtype=np.uint8
        )
        if inverse_y:
            expected_c_values = expected_c_values[::-1, :]
        np.testing.assert_almost_equal(expected_c_values,
                                       actual_c_values)

    def get_geo_data_frame_features(self):
        features = self.get_geo_json_features()
        return gpd.GeoDataFrame.from_features(features)

    @staticmethod
    def get_geo_json_features():
        feature1 = dict(
            type='Feature',
            geometry=dict(type='Polygon',
                          coordinates=[
                              [(-180, 0), (-1, 0), (-1, 90), (-180, 90),
                               (-180, 0)]]),
            properties=dict(a=0.5, b=2.1, c=9)
        )
        feature2 = dict(
            type='Feature',
            geometry=dict(type='Polygon',
                          coordinates=[
                              [(-180, -90), (-1, -90), (-1, 0), (-180, 0),
                               (-180, -90)]]),
            properties=dict(a=0.6, b=2.2, c=8)
        )
        feature3 = dict(
            type='Feature',
            geometry=dict(type='Polygon',
                          coordinates=[
                              [(20, 0), (180, 0), (180, 90), (20, 90),
                               (20, 0)]]),
            properties=dict(a=0.7, b=2.3, c=7)
        )
        feature4 = dict(
            type='Feature',
            geometry=dict(type='Polygon',
                          coordinates=[
                              [(20, -90), (180, -90), (180, 0), (20, 0),
                               (20, -90)]]),
            properties=dict(a=0.8, b=2.4, c=6)
        )
        return [feature1, feature2, feature3, feature4]


class DatasetGeometryTest(unittest.TestCase):

    def setUp(self) -> None:
        width = 16
        height = 8
        spatial_res = 360 / width
        lon_min = -2 * spatial_res + 0.5 * spatial_res
        lat_min = -2 * spatial_res + 0.5 * spatial_res
        lon_max = lon_min + 6 * spatial_res
        lat_max = lat_min + 3 * spatial_res

        self.triangle = shapely.geometry.Polygon(
            ((lon_min, lat_min),
             (lon_max, lat_min),
             (0.5 * (lon_max + lon_min), lat_max),
             (lon_min, lat_min))
        )

        self.cube = new_cube(width=width,
                             height=height,
                             x_res=spatial_res,
                             drop_bounds=True,
                             variables=dict(temp=273.9, precip=0.9))

    def test_clip_dataset_by_geometry(self):
        cube = clip_dataset_by_geometry(self.cube, self.triangle)
        self._assert_clipped_dataset_has_basic_props(cube)
        cube = clip_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_wkt=True)
        self._assert_clipped_dataset_has_basic_props(cube)
        self._assert_saved_geometry_wkt_is_fine(cube, 'geometry_wkt')
        cube = clip_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_wkt='intersect_geom')
        self._assert_saved_geometry_wkt_is_fine(cube, 'intersect_geom')

    def test_mask_dataset_by_geometry(self):
        cube = mask_dataset_by_geometry(self.cube, self.triangle)
        self._assert_clipped_dataset_has_basic_props(cube)
        cube = mask_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_wkt=True)
        self._assert_saved_geometry_wkt_is_fine(cube, 'geometry_wkt')
        cube = mask_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_wkt='intersect_geom')
        self._assert_saved_geometry_wkt_is_fine(cube, 'intersect_geom')

    def test_mask_dataset_by_geometry_excluded_vars(self):
        cube = mask_dataset_by_geometry(self.cube, self.triangle,
                                        excluded_vars='precip')
        self._assert_clipped_dataset_has_basic_props(cube)

    def test_mask_dataset_by_geometry_store_mask(self):
        cube = mask_dataset_by_geometry(self.cube, self.triangle,
                                        save_geometry_mask='geom_mask')
        self._assert_clipped_dataset_has_basic_props(cube)
        self._assert_dataset_mask_is_fine(cube, 'geom_mask')

    def test_clip_dataset_for_chunked_input(self):
        cube = chunk_dataset(self.cube,
                             chunk_sizes=dict(time=1, lat=90, lon=90))
        cube = clip_dataset_by_geometry(cube, self.triangle)
        self._assert_clipped_dataset_has_basic_props(cube)
        self.assertEqual(((1, 1, 1, 1, 1), (4,), (7,)), cube.temp.chunks)
        self.assertEqual(((1, 1, 1, 1, 1), (4,), (7,)), cube.precip.chunks)

    def test_clip_dataset_inverse_y(self):
        lat_inv = self.cube.lat[::-1]
        cube = self.cube.assign_coords(lat=lat_inv)
        cube = clip_dataset_by_geometry(cube, self.triangle)
        self._assert_clipped_dataset_has_basic_props(cube)

    def test_mask_dataset_for_chunked_input(self):
        cube = chunk_dataset(self.cube,
                             chunk_sizes=dict(time=1, lat=90, lon=90))
        cube = mask_dataset_by_geometry(cube, self.triangle)
        self._assert_clipped_dataset_has_basic_props(cube)
        self.assertEqual(((1, 1, 1, 1, 1), (4,), (7,)), cube.temp.chunks)
        self.assertEqual(((1, 1, 1, 1, 1), (4,), (7,)), cube.precip.chunks)

    def _assert_clipped_dataset_has_basic_props(self, dataset):
        self.assertEqual({'time': 5, 'lat': 4, 'lon': 7}, dataset.dims)
        self.assertIn('temp', dataset)
        self.assertIn('precip', dataset)
        temp = dataset['temp']
        precip = dataset['precip']
        self.assertEqual(('time', 'lat', 'lon'), temp.dims)
        self.assertEqual((5, 4, 7), temp.shape)
        self.assertEqual(('time', 'lat', 'lon'), precip.dims)
        self.assertEqual((5, 4, 7), precip.shape)

        self.assertIn('geospatial_lon_min', dataset.attrs)
        self.assertIn('geospatial_lon_max', dataset.attrs)
        self.assertIn('geospatial_lon_units', dataset.attrs)
        self.assertIn('geospatial_lon_resolution', dataset.attrs)
        self.assertIn('geospatial_lat_min', dataset.attrs)
        self.assertIn('geospatial_lat_max', dataset.attrs)
        self.assertIn('geospatial_lat_units', dataset.attrs)
        self.assertIn('geospatial_lat_resolution', dataset.attrs)
        self.assertIn('time_coverage_start', dataset.attrs)
        self.assertIn('time_coverage_end', dataset.attrs)
        self.assertIn('date_modified', dataset.attrs)

    def _assert_dataset_mask_is_fine(self, dataset, mask_var_name):
        self.assertIn(mask_var_name, dataset)
        geom_mask = dataset[mask_var_name]
        self.assertEqual(('lat', 'lon'), geom_mask.dims)
        self.assertEqual((4, 7), geom_mask.shape)
        actual_mask_values = geom_mask.values
        expected_mask_values = np.array(
            [
                [0, 1, 1, 1, 1, 1, 1],
                [0, 0, 1, 1, 1, 1, 0],
                [0, 0, 0, 1, 1, 0, 0],
                [0, 0, 0, 0, 0, 0, 0],
            ],
            dtype=np.bool
        )
        np.testing.assert_array_almost_equal(
            actual_mask_values,
            expected_mask_values
        )

    def _assert_saved_geometry_wkt_is_fine(self, dataset, geometry_wkt_name):
        self.assertIn(geometry_wkt_name, dataset.attrs)
        actual = shapely.wkt.loads(dataset.attrs[geometry_wkt_name])
        expected = shapely.wkt.loads(
            'POLYGON ((-33.75 -33.75, 33.75 33.75,'
            ' 101.25 -33.75, -33.75 -33.75))'
        )
        self.assertTrue(actual.difference(expected).is_empty)


class GetGeometryMaskTest(unittest.TestCase):
    # noinspection PyMethodMayBeStatic
    def test_get_geometry_mask(self):
        w = 16
        h = 8
        res = 1.0
        lon_min = 0
        lat_min = 0
        lon_max = lon_min + w * res
        lat_max = lat_min + h * res

        triangle = shapely.geometry.Polygon(((lon_min, lat_min),
                                             (lon_max, lat_min),
                                             (lon_max, lat_max),
                                             (lon_min, lat_min)))

        actual_mask = get_geometry_mask(w, h, triangle, lon_min,
                                        lat_min, res, all_touched=True)
        expected_mask = np.array(
            [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype=np.byte
        )
        np.testing.assert_array_almost_equal(expected_mask,
                                             actual_mask.astype('byte'))

        actual_mask = get_geometry_mask(w, h, triangle, lon_min,
                                        lat_min, res, all_touched=False)
        expected_mask = np.array(
            [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
             [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype=np.byte
        )
        np.testing.assert_array_almost_equal(expected_mask,
                                             actual_mask.astype('byte'))

        smaller_triangle = triangle.buffer(-1.5 * res)
        actual_mask = get_geometry_mask(w, h, smaller_triangle, lon_min,
                                        lat_min, res, all_touched=True)
        expected_mask = np.array(
            [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.byte
        )
        np.testing.assert_array_almost_equal(expected_mask, actual_mask)

        actual_mask = get_geometry_mask(w, h, smaller_triangle, lon_min,
                                        lat_min, res, all_touched=False)
        expected_mask = np.array(
            [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.byte
        )
        np.testing.assert_array_almost_equal(expected_mask, actual_mask)


class GetDatasetGeometryTest(unittest.TestCase):

    def test_nominal(self):
        ds1, ds2 = _get_nominal_datasets()
        bounds = get_dataset_geometry(ds1)
        self.assertEqual(
            shapely.geometry.box(-25.0, -15.0, 15.0, 15.0), bounds
        )
        bounds = get_dataset_geometry(ds2)
        self.assertEqual(
            shapely.geometry.box(-25.0, -15.0, 15.0, 15.0), bounds
        )

    def test_inv_y(self):
        ds1, ds2 = _get_inv_y_datasets()
        bounds = get_dataset_geometry(ds1)
        self.assertEqual(
            shapely.geometry.box(-25.0, -15.0, 15.0, 15.0), bounds
        )
        bounds = get_dataset_geometry(ds2)
        self.assertEqual(
            shapely.geometry.box(-25.0, -15.0, 15.0, 15.0), bounds
        )

    def test_antimeridian(self):
        ds1, ds2 = _get_antimeridian_datasets()
        bounds = get_dataset_geometry(ds1)
        self.assertEqual(shapely.geometry.MultiPolygon(
            polygons=[
                shapely.geometry.box(165.0, -15.0, 180.0, 15.0),
                shapely.geometry.box(-180.0, -15.0, -155.0, 15.0)
            ]),
            bounds)
        bounds = get_dataset_geometry(ds2)
        self.assertEqual(shapely.geometry.MultiPolygon(
            polygons=[
                shapely.geometry.box(165.0, -15.0, 180.0, 15.0),
                shapely.geometry.box(-180.0, -15.0, -155.0, 15.0)
            ]),
            bounds)


class GetDatasetBoundsTest(unittest.TestCase):
    def test_nominal(self):
        ds1, ds2 = _get_nominal_datasets()
        bounds = get_dataset_bounds(ds1)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
        bounds = get_dataset_bounds(ds2)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)

    def test_inv_y(self):
        ds1, ds2 = _get_inv_y_datasets()
        bounds = get_dataset_bounds(ds1)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
        bounds = get_dataset_bounds(ds2)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)

    def test_inv_y_wrong_order_bounds(self):
        ds1, ds2 = _get_inv_y_wrong_order_bounds_datasets()
        bounds = get_dataset_bounds(ds1)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
        bounds = get_dataset_bounds(ds2)
        self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)

    def test_anti_meridian(self):
        ds1, ds2 = _get_antimeridian_datasets()
        bounds = get_dataset_bounds(ds1)
        self.assertEqual((165.0, -15.0, -155.0, 15.0), bounds)
        bounds = get_dataset_bounds(ds2)
        self.assertEqual((165.0, -15.0, -155.0, 15.0), bounds)


def _get_nominal_datasets():
    data_vars = dict(a=(("time", "lat", "lon"), np.random.rand(5, 3, 4)))

    coords = dict(time=(("time",), np.array(range(0, 5))),
                  lat=(("lat",), np.array([-10, 0., 10])),
                  lon=(("lon",), np.array([-20, -10, 0., 10])))
    ds1 = xr.Dataset(coords=coords, data_vars=data_vars)

    # noinspection PyTypeChecker
    coords.update(
        lat_bnds=(
            ("lat", "bnds"),
            np.array([[-15, -5], [-5., 5], [5, 15]])
        ),
        lon_bnds=(
            ("lon", "bnds"),
            np.array([[-25., -15.], [-15., -5.], [-5., 5.], [5., 15.]])
        )
    )
    ds2 = xr.Dataset(coords=coords, data_vars=data_vars)

    return ds1, ds2


def _get_inv_y_datasets():
    ds1, ds2 = _get_nominal_datasets()
    ds1 = ds1.assign_coords(lat=(("lat",), ds1.lat.values[::-1]))
    ds2 = ds2.assign_coords(lat=(("lat",), ds1.lat.values))
    ds2 = ds2.assign_coords(lat_bnds=(("lat", "bnds"),
                                      ds2.lat_bnds.values[::-1, ::-1]))
    return ds1, ds2


def _get_inv_y_wrong_order_bounds_datasets():
    ds1, ds2 = _get_nominal_datasets()
    ds1 = ds1.assign_coords(lat=(("lat",), ds1.lat.values[::-1]))
    ds2 = ds2.assign_coords(lat=(("lat",), ds1.lat.values))
    ds2 = ds2.assign_coords(lat_bnds=(("lat", "bnds"),
                                      ds2.lat_bnds.values[::-1, ::]))
    return ds1, ds2


def _get_antimeridian_datasets():
    ds1, ds2 = _get_nominal_datasets()
    ds1 = ds1.assign_coords(lon=(("lon",),
                                 np.array([170., 180., -170., -160.])))
    ds2 = ds2.assign_coords(lon=(("lon",),
                                 ds1.lon.values))
    ds2 = ds2.assign_coords(
        lon_bnds=(("lon", "bnds"),
                  np.array([[165., 175], [175., -175.],
                            [-175., -165], [-165., -155.]])))
    return ds1, ds2


class NormalizeGeometryTest(unittest.TestCase):
    def test_normalize_null(self):
        self.assertIs(None, normalize_geometry(None))

    def test_normalize_to_point(self):
        expected_point = shapely.geometry.Point(12.8, -34.4)
        self.assertIs(expected_point,
                      normalize_geometry(expected_point))
        self.assertEqual(expected_point,
                         normalize_geometry([12.8, -34.4]))
        self.assertEqual(expected_point,
                         normalize_geometry(np.array([12.8, -34.4])))
        self.assertEqual(expected_point,
                         normalize_geometry(expected_point.wkt))
        self.assertEqual(expected_point,
                         normalize_geometry(expected_point.__geo_interface__))

    def test_normalize_box_as_point(self):
        expected_point = shapely.geometry.Point(12.8, -34.4)
        self.assertEqual(expected_point,
                         normalize_geometry([12.8, -34.4, 12.8, -34.4]))

    def test_normalize_to_box(self):
        expected_box = shapely.geometry.box(12.8, -34.4, 14.2, 20.6)
        self.assertIs(expected_box,
                      normalize_geometry(expected_box))
        self.assertEqual(expected_box,
                         normalize_geometry([12.8, -34.4, 14.2, 20.6]))
        self.assertEqual(expected_box,
                         normalize_geometry(
                             np.array([12.8, -34.4, 14.2, 20.6])
                         ))
        self.assertEqual(expected_box,
                         normalize_geometry(expected_box.wkt))
        self.assertEqual(expected_box,
                         normalize_geometry(expected_box.__geo_interface__))

    def test_normalize_to_split_box(self):
        expected_split_box = shapely.geometry.MultiPolygon(polygons=[
            shapely.geometry.Polygon(
                ((180.0, -34.4), (180.0, 20.6), (172.1, 20.6), (172.1, -34.4),
                 (180.0, -34.4))
            ),
            shapely.geometry.Polygon(
                ((-165.7, -34.4), (-165.7, 20.6), (-180.0, 20.6),
                 (-180.0, -34.4),
                 (-165.7, -34.4)))]
        )
        self.assertEqual(expected_split_box,
                         normalize_geometry([172.1, -34.4, -165.7, 20.6]))

    def test_normalize_from_geo_json_feature_dict(self):
        expected_box1 = shapely.geometry.box(-10, -20, 20, 10)
        expected_box2 = shapely.geometry.box(30, 20, 50, 40)
        feature1 = dict(type='Feature',
                        geometry=expected_box1.__geo_interface__)
        feature2 = dict(type='Feature',
                        geometry=expected_box2.__geo_interface__)
        feature_collection = dict(type='FeatureCollection',
                                  features=(feature1, feature2))

        actual_geom = normalize_geometry(feature1)
        self.assertEqual(expected_box1, actual_geom)

        actual_geom = normalize_geometry(feature2)
        self.assertEqual(expected_box2, actual_geom)

        expected_geom = shapely.geometry.GeometryCollection(
            geoms=[expected_box1, expected_box2]
        )
        actual_geom = normalize_geometry(feature_collection)
        self.assertEqual(expected_geom, actual_geom)

    def test_normalize_invalid_box(self):
        from xcube.core.geom import _INVALID_BOX_COORDS_MSG

        with self.assertRaises(ValueError) as cm:
            normalize_geometry([12.8, 20.6, 14.2, -34.4])
        self.assertEqual(_INVALID_BOX_COORDS_MSG, f'{cm.exception}')
        with self.assertRaises(ValueError) as cm:
            normalize_geometry([12.8, -34.4, 12.8, 20.6])
        self.assertEqual(_INVALID_BOX_COORDS_MSG, f'{cm.exception}')
        with self.assertRaises(ValueError) as cm:
            normalize_geometry([12.8, -34.4, 12.8, 20.6])
        self.assertEqual(_INVALID_BOX_COORDS_MSG, f'{cm.exception}')

    def test_invalid(self):
        from xcube.core.geom import _INVALID_GEOMETRY_MSG

        with self.assertRaises(ValueError) as cm:
            normalize_geometry(dict(coordinates=[12.8, -34.4]))
        self.assertEqual(_INVALID_GEOMETRY_MSG, f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            normalize_geometry([12.8, -34.4, '?'])
        self.assertEqual(_INVALID_GEOMETRY_MSG, f'{cm.exception}')

    def test_deprecated(self):
        self.assertEqual(None, convert_geometry(None))


class HelpersTest(unittest.TestCase):
    def test_is_lon_lat_dataset(self):
        dataset = new_cube(x_name='lon', y_name='lat')
        self.assertTrue(is_lon_lat_dataset(dataset))

        dataset = new_cube(x_name='x', y_name='y')
        self.assertTrue(is_lon_lat_dataset(dataset))

        dataset = new_cube(x_name='x', y_name='y', x_units='meters')
        self.assertFalse(is_lon_lat_dataset(dataset))
        dataset.x.attrs.update(long_name='longitude')
        dataset.y.attrs.update(long_name='latitude')
        self.assertTrue(is_lon_lat_dataset(dataset))

    def test_is_dataset_y_axis_inverted(self):
        dataset = new_cube()
        self.assertTrue(is_dataset_y_axis_inverted(dataset))

        dataset = new_cube(inverse_y=True)
        self.assertFalse(is_dataset_y_axis_inverted(dataset))


# test/core/test_level.py

import os
import time
import unittest

import numpy as np
import xarray as xr

from xcube.core.dsio import rimraf
from xcube.core.level import compute_levels
from xcube.core.level import read_levels
from xcube.core.level import write_levels


def get_path(filename):
    return os.path.join(os.path.dirname(__file__), filename)


class PyramidTest(unittest.TestCase):
    @classmethod
    def create_test_dataset(cls, shape, chunks=None):
        size = int(np.prod(shape))
        dims = ["time", "y", "x"]
        a_data = np.linspace(0, 1, size, dtype=np.float64).reshape(shape)
        a = xr.DataArray(a_data, dims=dims)
        b_data = np.linspace(-1, 0, size, dtype=np.float64).reshape(shape)
        b = xr.DataArray(b_data, dims=dims)
        if chunks:
            a.encoding.update(chunks=chunks, chunksizes=chunks)
            b.encoding.update(chunks=chunks, chunksizes=chunks)
            return xr.Dataset(dict(a=a, b=b)) \
                .chunk(chunks={dims[i]: chunks[i]
                               for i in range(len(dims))})
        else:
            return xr.Dataset(dict(a=a, b=b))

    def test_compute_levels(self):
        dataset = self.create_test_dataset(shape=(5, 200, 400),
                                           chunks=(1, 25, 25))
        levels = compute_levels(dataset)
        self._assert_levels_ok(levels,
                               expected_num_levels=4,
                               expected_shapes=[
                                   (5, 200, 400),
                                   (5, 100, 200),
                                   (5, 50, 100),
                                   (5, 25, 50),
                               ],
                               expected_chunks=[
                                   ((1,) * 5, (25,) * 8, (25,) * 16),
                                   ((1,) * 5, (25,) * 4, (25,) * 8),
                                   ((1,) * 5, (25,) * 2, (25,) * 4),
                                   ((1,) * 5, (25,) * 1, (25,) * 2),
                               ])

    def test_compute_levels_with_max_levels(self):
        dataset = self.create_test_dataset(shape=(5, 200, 400),
                                           chunks=(1, 25, 25))
        levels = compute_levels(dataset, num_levels_max=3)
        self._assert_levels_ok(levels,
                               expected_num_levels=3,
                               expected_shapes=[
                                   (5, 200, 400),
                                   (5, 100, 200),
                                   (5, 50, 100),
                               ],
                               expected_chunks=[
                                   ((1,) * 5, (25,) * 8, (25,) * 16),
                                   ((1,) * 5, (25,) * 4, (25,) * 8),
                                   ((1,) * 5, (25,) * 2, (25,) * 4),
                               ])

    def test_write_read_levels_with_even_sizes(self):
        shape = (5, 200, 400)
        tile_shape = (25, 25)
        expected_num_levels = 4
        expected_shapes = [
            (5, 200, 400),
            (5, 100, 200),
            (5, 50, 100),
            (5, 25, 50),
        ]
        expected_chunks = [
            ((1,) * 5, (25,) * 8, (25,) * 16),
            ((1,) * 5, (25,) * 4, (25,) * 8),
            ((1,) * 5, (25,) * 2, (25,) * 4),
            ((1,) * 5, (25,) * 1, (25,) * 2),
        ]
        self._assert_io_ok(shape,
                           tile_shape,
                           False,
                           expected_num_levels,
                           expected_shapes,
                           expected_chunks)
        self._assert_io_ok(shape,
                           tile_shape,
                           True,
                           expected_num_levels,
                           expected_shapes,
                           expected_chunks)

    def test_write_read_levels_with_odd_sizes(self):
        shape = (5, 203, 405)
        tile_shape = (37, 38)
        expected_num_levels = 3
        expected_shapes = [
            (5, 203, 405),
            (5, 102, 203),
            (5, 51, 102),
        ]
        expected_chunks = [
            ((1,) * 5,
             (37, 37, 37, 37, 37, 18),
             (38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 25)),
            ((1,) * 5,
             (37, 37, 28),
             (38, 38, 38, 38, 38, 13)),
            ((1,) * 5,
             (37, 14),
             (38, 38, 26)),
        ]
        self._assert_io_ok(shape,
                           tile_shape,
                           False,
                           expected_num_levels,
                           expected_shapes,
                           expected_chunks)
        self._assert_io_ok(shape,
                           tile_shape,
                           True,
                           expected_num_levels,
                           expected_shapes,
                           expected_chunks)

    def _assert_io_ok(self,
                      shape,
                      tile_shape,
                      link_input: bool,
                      expected_num_levels,
                      expected_shapes,
                      expected_chunks):

        input_path = get_path("pyramid-input.zarr")
        output_path = get_path("pyramid-output")

        rimraf(input_path)
        rimraf(output_path)

        try:
            dataset = self.create_test_dataset(shape,
                                               chunks=(1, *tile_shape))
            dataset.to_zarr(input_path)

            t0 = time.perf_counter()

            levels = write_levels(output_path,
                                  dataset=dataset,
                                  spatial_tile_shape=tile_shape,
                                  input_path=input_path,
                                  link_input=link_input)

            print(f"write time total: ", time.perf_counter() - t0)

            self._assert_levels_ok(levels,
                                   expected_num_levels,
                                   expected_shapes,
                                   expected_chunks)

            t0 = time.perf_counter()

            levels = read_levels(output_path)

            print(f"read time total: ", time.perf_counter() - t0)

            self._assert_levels_ok(levels,
                                   expected_num_levels,
                                   expected_shapes,
                                   expected_chunks)

        finally:
            rimraf(input_path)
            rimraf(output_path)

    def _assert_levels_ok(self,
                          levels,
                          expected_num_levels,
                          expected_shapes,
                          expected_chunks):
        self.assertIsInstance(levels, list)
        self.assertEqual(expected_num_levels, len(levels))

        for i in range(expected_num_levels):
            msg = f"at index {i}"
            self.assertIsInstance(levels[i], xr.Dataset, msg=msg)
            self.assertEqual(2, len(levels[i].data_vars), msg=msg)
            self.assertIn("a", levels[i], msg=msg)
            self.assertIn("b", levels[i], msg=msg)

        for i in range(expected_num_levels):
            msg = f"at index {i}"
            self.assertEqual(expected_shapes[i], levels[i]["a"].shape,
                             msg=msg)
            self.assertEqual(expected_shapes[i], levels[i]["b"].shape,
                             msg=msg)

        for i in range(expected_num_levels):
            msg = f"at index {i}"
            self.assertEqual(expected_chunks[i], levels[i]["a"].chunks,
                             msg=msg)
            self.assertEqual(expected_chunks[i], levels[i]["b"].chunks,
                             msg=msg)


# test/core/test_maskset.py

import unittest
from typing import Tuple

import numpy as np
import xarray as xr
from numpy.testing import assert_array_almost_equal

from test.sampledata import create_highroc_dataset, create_c2rcc_flag_var, create_cmems_sst_flag_var
from xcube.core.maskset import MaskSet


class MaskSetTest(unittest.TestCase):
    def test_mask_set_with_flag_mask_str(self):
        flag_var = create_cmems_sst_flag_var().chunk(dict(lon=2, lat=2))
        mask_set = MaskSet(flag_var)

        self.assertEqual('mask(sea=(1, None), land=(2, None), lake=(4, None), ice=(8, None))',
                         str(mask_set))

        expected_chunks = ((1,), (2, 1), (2, 2))
        self.assertMaskOk(mask_set, 'sea', expected_chunks)
        self.assertMaskOk(mask_set, 'land', expected_chunks)
        self.assertMaskOk(mask_set, 'lake', expected_chunks)
        self.assertMaskOk(mask_set, 'ice', expected_chunks)

        validation_data = ((0, 'sea', mask_set.sea, np.array([[[1, 0, 0, 0],
                                                               [1, 1, 0, 0],
                                                               [1, 1, 1, 0]]],
                                                             dtype=np.uint8)),
                           (1, 'land', mask_set.land, np.array([[[0, 1, 0, 0],
                                                                 [0, 0, 1, 1],
                                                                 [0, 0, 0, 1]]],
                                                               dtype=np.uint8)),
                           (2, 'lake', mask_set.lake, np.array([[[0, 0, 1, 1],
                                                                 [0, 0, 0, 0],
                                                                 [0, 0, 0, 0]]],
                                                               dtype=np.uint8)),
                           (3, 'ice', mask_set.ice, np.array([[[1, 1, 1, 0],
                                                               [1, 0, 0, 0],
                                                               [0, 0, 0, 0]]],
                                                             dtype=np.uint8)))

        for index, name, mask, data in validation_data:
            self.assertIs(mask, mask_set[index])
            self.assertIs(mask, mask_set[name])
            assert_array_almost_equal(mask.values, data, err_msg=f'{index}, {name}, {mask.name}')

    def test_mask_set_with_flag_mask_int_array(self):
        flag_var = create_c2rcc_flag_var().chunk(dict(x=2, y=2))
        mask_set = MaskSet(flag_var)

        self.assertEqual('c2rcc_flags(F1=(1, None), F2=(2, None), F3=(4, None), F4=(8, None))',
                         str(mask_set))

        expected_chunks = ((2, 1), (2, 2))
        self.assertMaskOk(mask_set, 'F1', expected_chunks)
        self.assertMaskOk(mask_set, 'F2', expected_chunks)
        self.assertMaskOk(mask_set, 'F3', expected_chunks)
        self.assertMaskOk(mask_set, 'F4', expected_chunks)

        validation_data = ((0, 'F1', mask_set.F1, np.array([[1, 1, 1, 1],
                                                            [1, 0, 1, 0],
                                                            [0, 1, 1, 1]],
                                                           dtype=np.uint8)),
                           (1, 'F2', mask_set.F2, np.array([[0, 0, 0, 0],
                                                            [0, 0, 0, 1],
                                                            [0, 0, 0, 0]],
                                                           dtype=np.uint8)),
                           (2, 'F3', mask_set.F3, np.array([[0, 0, 0, 0],
                                                            [0, 1, 0, 0],
                                                            [0, 0, 0, 0]],
                                                           dtype=np.uint8)),
                           (3, 'F4', mask_set.F4, np.array([[0, 0, 0, 0],
                                                            [0, 0, 0, 0],
                                                            [1, 0, 0, 0]],
                                                           dtype=np.uint8)))

        for index, name, mask, data in validation_data:
            self.assertIs(mask, mask_set[index])
            self.assertIs(mask, mask_set[name])
            assert_array_almost_equal(mask.values, data)

    def assertMaskOk(self,
                     mask_set: MaskSet,
                     mask_name: str,
                     expected_chunks: Tuple[Tuple[int, ...], ...] = None):

        mask = getattr(mask_set, mask_name)

        self.assertIsInstance(mask, xr.DataArray)

        if expected_chunks:
            import dask.array as da
            self.assertIsInstance(mask.data, da.Array)
        self.assertEqual(expected_chunks, mask.chunks)

        # assert same instance is returned
        self.assertIs(mask, getattr(mask_set, mask_name))

        self.assertIn(mask_name, mask_set)
        # assert same instance is returned
        self.assertIs(mask, mask_set[mask_name])

    def test_get_mask_sets(self):
        dataset = create_highroc_dataset()
        mask_sets = MaskSet.get_mask_sets(dataset)
        self.assertIsNotNone(mask_sets)
        self.assertEqual(len(mask_sets), 1)
        self.assertIn('c2rcc_flags', mask_sets)
        mask_set = mask_sets['c2rcc_flags']
        self.assertIsInstance(mask_set, MaskSet)

    def test_mask_set_with_flag_values(self):
        s2l2a_slc_meanings = ['no_data',
                              'saturated_or_defective',
                              'dark_area_pixels',
                              'cloud_shadows',
                              'vegetation',
                              'bare_soils',
                              'water',
                              'clouds_low_probability_or_unclassified',
                              'clouds_medium_probability',
                              'clouds_high_probability',
                              'cirrus',
                              'snow_or_ice']

        data = np.array([[1, 2, 8, 3],
                         [7, 6, 0, 4],
                         [9, 5, 11, 10]], dtype=np.uint8)
        flag_var = xr.DataArray(data,
                                dims=('y', 'x'),
                                name='SLC',
                                attrs=dict(
                                    long_name="Scene classification flags",
                                    flag_values=','.join(f'{i}' for i in range(len(s2l2a_slc_meanings))),
                                    flag_meanings=' '.join(s2l2a_slc_meanings),
                                ))

        mask_set = MaskSet(flag_var)

        self.assertEqual('SLC(no_data=(None, 0), saturated_or_defective=(None, 1), dark_area_pixels=(None, 2), '
                         'cloud_shadows=(None, 3), vegetation=(None, 4), bare_soils=(None, 5), water=(None, 6), '
                         'clouds_low_probability_or_unclassified=(None, 7), clouds_medium_probability=(None, 8), '
                         'clouds_high_probability=(None, 9), cirrus=(None, 10), snow_or_ice=(None, 11))',
                         str(mask_set))

        validation_data = ((0, 'no_data', mask_set.no_data, np.array([[0, 0, 0, 0],
                                                                      [0, 0, 1, 0],
                                                                      [0, 0, 0, 0]],
                                                                     dtype=np.uint8)),
                           (4, 'vegetation', mask_set.vegetation, np.array([[0, 0, 0, 0],
                                                                            [0, 0, 0, 1],
                                                                            [0, 0, 0, 0]],
                                                                           dtype=np.uint8)),
                           (10, 'cirrus', mask_set.cirrus, np.array([[0, 0, 0, 0],
                                                                     [0, 0, 0, 0],
                                                                     [0, 0, 0, 1]],
                                                                    dtype=np.uint8)),
                           (6, 'water', mask_set.water, np.array([[0, 0, 0, 0],
                                                                  [0, 1, 0, 0],
                                                                  [0, 0, 0, 0]],
                                                                 dtype=np.uint8)))

        for index, name, mask, data in validation_data:
            msg = f'index={index}, name={name!r}, data={data!r}'
            self.assertIs(mask, mask_set[index], msg=msg)
            self.assertIs(mask, mask_set[name], msg=msg)
            assert_array_almost_equal(mask.values, data, err_msg=msg)

        self.assertEqual(set(s2l2a_slc_meanings), set(dir(mask_set)))

        html = mask_set._repr_html_()
        self.assertTrue(html.startswith('<html>'))
        self.assertTrue(html.endswith('</html>'))

    def test_mask_set_with_missing_values_and_masks_attrs(self):
        flag_var = create_c2rcc_flag_var().chunk(dict(x=2, y=2))
        flag_var.attrs.pop('flag_masks', None)
        flag_var.attrs.pop('flag_values', None)
        with self.assertRaises(ValueError):
            MaskSet(flag_var)

    def test_mask_set_with_missing_meanings_attr(self):
        flag_var = create_c2rcc_flag_var().chunk(dict(x=2, y=2))
        flag_var.attrs.pop('flag_meanings', None)
        with self.assertRaises(ValueError):
            MaskSet(flag_var)


# test/core/test_mldataset.py

import os
import unittest

import numpy as np
import pandas as pd
import s3fs
import xarray as xr

from test.s3test import S3Test, MOTO_SERVER_ENDPOINT_URL
from xcube.core.dsio import write_dataset
from xcube.core.geom import clip_dataset_by_geometry
from xcube.core.mldataset import BaseMultiLevelDataset
from xcube.core.mldataset import CombinedMultiLevelDataset
from xcube.core.mldataset import ComputedMultiLevelDataset
from xcube.core.mldataset import ObjectStorageMultiLevelDataset
from xcube.core.mldataset import open_ml_dataset_from_object_storage
from xcube.core.mldataset import write_levels
from xcube.core.tilingscheme import TilingScheme
from xcube.util.tilegrid import ImageTileGrid


class CombinedMultiLevelDatasetTest(unittest.TestCase):
    def test_it(self):
        ml_ds_1 = BaseMultiLevelDataset(
            _get_test_dataset(('noise_1', 'noise_2'))
        )
        ml_ds_2 = BaseMultiLevelDataset(
            _get_test_dataset(('noise_3', 'noise_4'))
        )
        ml_ds_3 = BaseMultiLevelDataset(
            _get_test_dataset(('noise_5', 'noise_6'))
        )

        ml_ds = CombinedMultiLevelDataset([ml_ds_1, ml_ds_2, ml_ds_3])

        self.assertEqual(3, ml_ds.num_levels)
        self.assertIsInstance(ml_ds.tile_grid, ImageTileGrid)
        self.assertEqual((180, 180), ml_ds.tile_grid.tile_size)
        self.assertEqual(3, ml_ds.tile_grid.num_levels)

        expected_var_names = {'noise_1', 'noise_2',
                              'noise_3', 'noise_4',
                              'noise_5', 'noise_6'}

        ds0 = ml_ds.get_dataset(0)
        self.assertEqual({'time': 14, 'lat': 720, 'lon': 1440, 'bnds': 2},
                         ds0.dims)
        self.assertEqual(expected_var_names, set(map(str, ds0.data_vars)))
        self.assertTrue(all(v.dims == ('time', 'lat', 'lon')
                            for v in ds0.data_vars.values()))

        ds1 = ml_ds.get_dataset(1)
        self.assertEqual({'time': 14, 'lat': 360, 'lon': 720},
                         ds1.dims)
        self.assertEqual(expected_var_names, set(map(str, ds1.data_vars)))
        self.assertTrue(all(v.dims == ('time', 'lat', 'lon')
                            for v in ds1.data_vars.values()))

        ds2 = ml_ds.get_dataset(2)
        self.assertEqual({'time': 14, 'lat': 180, 'lon': 360},
                         ds2.dims)
        self.assertEqual(expected_var_names, set(map(str, ds2.data_vars)))
        self.assertTrue(all(v.dims == ('time', 'lat', 'lon')
                            for v in ds2.data_vars.values()))

        self.assertEqual([ds0, ds1, ds2], ml_ds.datasets)

        ml_ds.close()


class BaseMultiLevelDatasetTest(unittest.TestCase):
    def test_basic_props(self):
        ds = _get_test_dataset()
        ml_ds = BaseMultiLevelDataset(ds)

        self.assertIsInstance(ml_ds.ds_id, str)

        self.assertEqual(3, ml_ds.num_levels)
        self.assertEqual([(0.25, 0.25), (0.5, 0.5), (1.0, 1.0)],
                         ml_ds.resolutions)
        self.assertEqual([0.25, 0.5, 1.0],
                         ml_ds.avg_resolutions)

        tile_grid = ml_ds.tile_grid
        self.assertIsInstance(tile_grid, ImageTileGrid)
        self.assertEqual((180, 180), tile_grid.tile_size)
        self.assertEqual(3, tile_grid.num_levels)

    def test_level_datasets(self):
        ds = _get_test_dataset()
        ml_ds = BaseMultiLevelDataset(ds)

        ds0 = ml_ds.get_dataset(0)
        self.assertIsNot(ds, ds0)
        self.assertEqual({'time': 14, 'lat': 720, 'lon': 1440, 'bnds': 2},
                         ds0.dims)

        ds1 = ml_ds.get_dataset(1)
        self.assertIsNot(ds, ds1)
        self.assertEqual({'time': 14, 'lat': 360, 'lon': 720},
                         ds1.dims)

        ds2 = ml_ds.get_dataset(2)
        self.assertIsNot(ds, ds2)
        self.assertEqual({'time': 14, 'lat': 180, 'lon': 360},
                         ds2.dims)

        self.assertEqual([ds0, ds1, ds2], ml_ds.datasets)

        ml_ds.close()

    def test_arg_validation(self):
        ds = _get_test_dataset()
        with self.assertRaises(TypeError):
            # noinspection PyTypeChecker
            BaseMultiLevelDataset('test.levels')
        with self.assertRaises(TypeError):
            # noinspection PyTypeChecker
            BaseMultiLevelDataset(ds, tile_grid=512)
        with self.assertRaises(TypeError):
            # noinspection PyTypeChecker
            BaseMultiLevelDataset(ds, grid_mapping='crs84')

    def test_derive_tiling_scheme(self):
        ds = _get_test_dataset()
        ml_ds = BaseMultiLevelDataset(ds)
        tiling_scheme = ml_ds.derive_tiling_scheme(TilingScheme.GEOGRAPHIC)
        self.assertEqual('CRS84', tiling_scheme.crs_name)
        self.assertEqual(0, tiling_scheme.min_level)
        self.assertEqual(2, tiling_scheme.max_level)


class ComputedMultiLevelDatasetTest(unittest.TestCase):
    def test_it(self):
        ds = _get_test_dataset()

        ml_ds1 = BaseMultiLevelDataset(ds)

        def input_ml_dataset_getter(ds_id):
            if ds_id == "ml_ds1":
                return ml_ds1
            self.fail(f"unexpected ds_id={ds_id!r}")

        ml_ds2 = ComputedMultiLevelDataset(
            os.path.join(os.path.dirname(__file__),
                         "..", "webapi", "res", "test", "script.py"),
            "compute_dataset",
            ["ml_ds1"],
            input_ml_dataset_getter,
            input_parameters=dict(period='1W'),
            ds_id="ml_ds2"
        )
        self.assertEqual(3, ml_ds2.num_levels)
        self.assertIsInstance(ml_ds2.tile_grid, ImageTileGrid)
        self.assertEqual((180, 180), ml_ds2.tile_grid.tile_size)
        self.assertEqual(3, ml_ds2.tile_grid.num_levels)

        ds0 = ml_ds2.get_dataset(0)
        self.assertEqual({'time': 3, 'lat': 720, 'lon': 1440, 'bnds': 2}, ds0.dims)

        ds1 = ml_ds2.get_dataset(1)
        self.assertEqual({'time': 3, 'lat': 360, 'lon': 720}, ds1.dims)

        ds2 = ml_ds2.get_dataset(2)
        self.assertEqual({'time': 3, 'lat': 180, 'lon': 360}, ds2.dims)

        self.assertEqual([ds0, ds1, ds2], ml_ds2.datasets)

        ml_ds1.close()
        ml_ds2.close()


def _get_test_dataset(var_names=('noise',)):
    w = 1440
    h = 720
    p = 14

    x1 = -180.
    y1 = -90.
    x2 = +180.
    y2 = +90.
    dx = (x2 - x1) / w
    dy = (y2 - y1) / h

    data_vars = {var_name: (("time", "lat", "lon"), np.random.rand(p, h, w)) for var_name in var_names}

    lat_bnds = np.array(list(zip(np.linspace(y2, y1 + dy, num=h),
                                 np.linspace(y2 - dy, y1, num=h))))
    lon_bnds = np.array(list(zip(np.linspace(x1, x2 - dx, num=w),
                                 np.linspace(x1 + x2, x2, num=w))))
    coords = dict(time=(("time",),
                        np.array(pd.date_range(start="2019-01-01T12:00:00Z", periods=p, freq="1D"),
                                 dtype="datetime64[ns]")),
                  lat=(("lat",),
                       np.linspace(y2 - .5 * dy, y1 + .5 * dy, num=h)),
                  lon=(("lon",),
                       np.linspace(x1 + .5 * dx, x2 - .5 * dx, num=w)),
                  lat_bnds=(("lat", "bnds"), lat_bnds),
                  lon_bnds=(("lon", "bnds"), lon_bnds))

    return xr.Dataset(coords=coords, data_vars=data_vars).chunk(chunks=dict(lat=180, lon=180))


class ObjectStorageMultiLevelDatasetTest(S3Test):
    def test_s3_zarr(self):
        # if this test fails on travis with
        # E           IndexError: pop from an empty deque
        # then it most likly might be due to this issue: https://github.com/pydata/xarray/issues/4478
        # The issue has been fixed, but dask/s3fs does not have a new release including the fix yet

        self._write_test_cube()

        ml_ds_from_object_storage = open_ml_dataset_from_object_storage(
            'xcube-test-cube/cube-1-250-250.zarr',
            s3_kwargs=dict(key='test_fake_id', secret='test_fake_secret'),
            s3_client_kwargs=dict(endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        )
        self.assertIsNotNone(ml_ds_from_object_storage)
        self.assertIn('conc_chl', ml_ds_from_object_storage.base_dataset.variables)
        self.assertEqual((2, 200, 400), ml_ds_from_object_storage.base_dataset.conc_chl.shape)

    @classmethod
    def _write_test_cube(cls):
        s3_kwargs = dict(key='test_fake_id', secret='test_fake_secret')
        s3_client_kwargs = dict(endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        s3 = s3fs.S3FileSystem(**s3_kwargs, client_kwargs=s3_client_kwargs)
        # Create bucket 'xcube-test', so it exists before we write a test cube
        s3.mkdir('xcube-test-cube')

        # Create a test cube with just one variable "conc_chl"
        zarr_path = os.path.join(os.path.dirname(__file__), '../../examples/serve/demo/cube-1-250-250.zarr')
        dataset = xr.open_zarr(zarr_path)
        dataset = cls._make_subset(dataset)

        # Write test cube
        write_dataset(dataset,
                      'xcube-test-cube/cube-1-250-250.zarr',
                      s3_kwargs=s3_kwargs,
                      s3_client_kwargs=s3_client_kwargs)

    def test_s3_levels(self):
        self._write_test_cube_pyramid()

        s3 = s3fs.S3FileSystem(key='test_fake_id',
                               secret='test_fake_secret',
                               client_kwargs=dict(endpoint_url=MOTO_SERVER_ENDPOINT_URL))
        ml_dataset = ObjectStorageMultiLevelDataset(s3,
                                                    "xcube-test/cube-1-250-250.levels",
                                                    chunk_cache_capacity=1000 * 1000 * 1000)
        self.assertIsNotNone(ml_dataset)
        self.assertEqual(2, ml_dataset.num_levels)
        self.assertEqual((100, 100), ml_dataset.tile_grid.tile_size)
        self.assertEqual((2, 1), ml_dataset.tile_grid.get_num_tiles(0))
        self.assertEqual(800000000, ml_dataset.get_chunk_cache_capacity(0))
        self.assertEqual(200000000, ml_dataset.get_chunk_cache_capacity(1))

    @classmethod
    def _write_test_cube_pyramid(cls):
        s3_kwargs = dict(key='test_fake_id', secret='test_fake_secret')
        s3_client_kwargs = dict(endpoint_url=MOTO_SERVER_ENDPOINT_URL)
        s3 = s3fs.S3FileSystem(**s3_kwargs, client_kwargs=s3_client_kwargs)
        # Create bucket 'xcube-test', so it exists before we write a test pyramid
        s3.mkdir('xcube-test')

        # Create a test cube pyramid with just one variable "conc_chl"
        zarr_path = os.path.join(os.path.dirname(__file__), '../../examples/serve/demo/cube-1-250-250.zarr')
        base_dataset = xr.open_zarr(zarr_path)
        base_dataset = cls._make_subset(base_dataset)
        ml_dataset = BaseMultiLevelDataset(base_dataset)

        # Write test cube pyramid
        write_levels(ml_dataset,
                     'xcube-test/cube-1-250-250.levels',
                     s3_kwargs=s3_kwargs,
                     s3_client_kwargs=s3_client_kwargs)

    @classmethod
    def _make_subset(cls, base_dataset):
        base_dataset = base_dataset.drop_vars(['c2rcc_flags', 'conc_tsm', 'kd489', 'quality_flags'])
        geometry = (1.0, 51.0, 2.0, 51.5)
        base_dataset = clip_dataset_by_geometry(base_dataset, geometry=geometry)
        base_dataset = base_dataset.dropna('time', how='all')
        # somehow clip_dataset_by_geometry() does not unify chunks and encoding, so it needs to be done manually:
        for var in base_dataset.variables:
            if var not in base_dataset.coords:
                if base_dataset[var].encoding['chunks'] != (1, 100, 100):
                    base_dataset[var].encoding['chunks'] = (1, 100, 100)
                base_dataset[var] = base_dataset[var].chunk({'time': 1, 'lat': 100, 'lon': 100})
        base_dataset['lat'] = base_dataset['lat'].chunk({'lat': 100})
        base_dataset.lat.encoding['chunks'] = (100,)
        base_dataset['lon'] = base_dataset['lon'].chunk({'lon': 100})
        base_dataset.lon.encoding['chunks'] = (100,)
        base_dataset['lat_bnds'] = base_dataset['lat_bnds'].chunk((100, 2))
        base_dataset.lat_bnds.encoding['chunks'] = (100, 2)
        base_dataset['lon_bnds'] = base_dataset['lon_bnds'].chunk((100, 2))
        base_dataset.lon_bnds.encoding['chunks'] = (100, 2)
        return base_dataset


# test/core/test_new.py

import unittest

import numpy as np

from xcube.core.new import new_cube


class NewCubeTest(unittest.TestCase):

    def test_new_cube_with_bounds(self):
        cube = new_cube()
        self.assertEqual({'lon': 360, 'lat': 180, 'time': 5, 'bnds': 2}, cube.dims)
        self.assertEqual(-179.5, float(cube.lon[0]))
        self.assertEqual(179.5, float(cube.lon[-1]))
        self.assertEqual(-89.5, float(cube.lat[0]))
        self.assertEqual(89.5, float(cube.lat[-1]))
        self.assertEqual(-180., float(cube.lon_bnds[0, 0]))
        self.assertEqual(-179., float(cube.lon_bnds[0, 1]))
        self.assertEqual(179., float(cube.lon_bnds[-1, 0]))
        self.assertEqual(180., float(cube.lon_bnds[-1, 1]))
        self.assertEqual(-90., float(cube.lat_bnds[0, 0]))
        self.assertEqual(-89., float(cube.lat_bnds[0, 1]))
        self.assertEqual(89., float(cube.lat_bnds[-1, 0]))
        self.assertEqual(90., float(cube.lat_bnds[-1, 1]))

    def test_new_cube_without_bounds(self):
        cube = new_cube(drop_bounds=True)
        self.assertEqual({'lon': 360, 'lat': 180, 'time': 5}, cube.dims)

    def test_new_cube_with_const_vars(self):
        cube = new_cube(variables=dict(sst=274.4, chl=10.31))
        self.assertIn('sst', cube)
        self.assertIn('chl', cube)
        import numpy as np
        np.testing.assert_almost_equal(cube.sst.values, np.full((5, 180, 360), 274.4))
        np.testing.assert_almost_equal(cube.chl.values, np.full((5, 180, 360), 10.31))

    def test_new_cube_with_func_vars(self):
        def sst_func(t, y, x):
            return 274.4 + t * 0.5

        def chl_func(t, y, x):
            return 10.31 + y * 0.1

        def aot_func(t, y, x):
            return 0.88 + x * 0.05

        cube = new_cube(width=16, height=8, variables=dict(sst=sst_func, chl=chl_func, aot=aot_func))

        self.assertIn('sst', cube)
        self.assertIn('chl', cube)
        self.assertIn('aot', cube)

        np.testing.assert_almost_equal(cube.sst[0, :, :].values, np.full((8, 16), 274.4))
        np.testing.assert_almost_equal(cube.chl[:, 0, :].values, np.full((5, 16), 10.31))
        np.testing.assert_almost_equal(cube.aot[:, :, 0].values, np.full((5, 8), 0.88))

        np.testing.assert_almost_equal(cube.sst.isel(lon=3, lat=2).values,
                                       np.array([274.4, 274.9, 275.4, 275.9, 276.4]))
        np.testing.assert_almost_equal(cube.chl.isel(time=3, lon=4).values,
                                       np.array([10.31, 10.41, 10.51, 10.61, 10.71, 10.81, 10.91, 11.01]))
        np.testing.assert_almost_equal(cube.aot.isel(time=1, lat=2).values,
                                       np.array([0.88, 0.93, 0.98, 1.03, 1.08, 1.13, 1.18, 1.23,
                                                 1.28, 1.33, 1.38, 1.43, 1.48, 1.53, 1.58, 1.63]))


# test/core/test_normalize.py

"""
Test for the normalization operation
"""

from datetime import datetime
from unittest import TestCase

import numpy as np
import pandas as pd
import pyproj
import xarray as xr
from jdcal import gcal2jd
from numpy.testing import assert_array_almost_equal

from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.normalize import DatasetIsNotACubeError
from xcube.core.normalize import adjust_spatial_attrs
from xcube.core.normalize import decode_cube
from xcube.core.normalize import encode_cube
from xcube.core.normalize import normalize_coord_vars
from xcube.core.normalize import normalize_dataset
from xcube.core.normalize import normalize_missing_time


# noinspection PyPep8Naming
def assertDatasetEqual(expected, actual):
    # this method is functionally equivalent to
    # `assert expected == actual`, but it
    # checks each aspect of equality separately for easier debugging
    assert expected.equals(actual), (expected, actual)


class DecodeCubeTest(TestCase):
    def test_cube_stays_cube(self):
        dataset = new_cube(variables=dict(a=1, b=2, c=3))
        cube, grid_mapping, rest = decode_cube(dataset)
        self.assertIs(dataset, cube)
        self.assertIsInstance(grid_mapping, GridMapping)
        self.assertTrue(grid_mapping.crs.is_geographic)
        self.assertIsInstance(rest, xr.Dataset)
        self.assertEqual(set(), set(rest.data_vars))

    def test_no_cube_vars_are_dropped(self):
        dataset = new_cube(variables=dict(a=1, b=2, c=3))
        dataset = dataset.assign(
            d=xr.DataArray([8, 9, 10], dims='level'),
            crs=xr.DataArray(0, attrs=pyproj.CRS.from_string('CRS84').to_cf()),
        )
        self.assertEqual({'a', 'b', 'c', 'd', 'crs'}, set(dataset.data_vars))
        cube, grid_mapping, rest = decode_cube(dataset)
        self.assertIsInstance(cube, xr.Dataset)
        self.assertIsInstance(grid_mapping, GridMapping)
        self.assertEqual({'a', 'b', 'c'}, set(cube.data_vars))
        self.assertEqual(pyproj.CRS.from_string('CRS84'), grid_mapping.crs)
        self.assertIsInstance(rest, xr.Dataset)
        self.assertEqual({'d', 'crs'}, set(rest.data_vars))

    def test_encode_is_inverse(self):
        dataset = new_cube(variables=dict(a=1, b=2, c=3),
                           x_name='x', y_name='y')
        dataset = dataset.assign(
            d=xr.DataArray([8, 9, 10], dims='level'),
            crs=xr.DataArray(0, attrs=pyproj.CRS.from_string('CRS84').to_cf()),
        )
        cube, grid_mapping, rest = decode_cube(dataset)
        dataset2 = encode_cube(cube, grid_mapping, rest)
        self.assertEqual(set(dataset.data_vars), set(dataset2.data_vars))
        self.assertIn('crs', dataset2.data_vars)

    def test_no_cube_vars_found(self):
        dataset = new_cube()
        self.assertEqual(set(), set(dataset.data_vars))
        with self.assertRaises(DatasetIsNotACubeError) as cm:
            decode_cube(dataset, force_non_empty=True)
        self.assertEqual("No variables found with dimensions"
                         " ('time', [...] 'lat', 'lon')"
                         " or dimension sizes too small",
                         f'{cm.exception}')

    def test_no_grid_mapping(self):
        dataset = xr.Dataset(dict(a=[1, 2, 3], b=0.5))
        with self.assertRaises(DatasetIsNotACubeError) as cm:
            decode_cube(dataset)
        self.assertEqual("Failed to detect grid mapping:"
                         " cannot find any grid mapping in dataset",
                         f'{cm.exception}')

    def test_grid_mapping_not_geographic(self):
        dataset = new_cube(x_name='x', y_name='y',
                           variables=dict(a=0.5), crs='epsg:25832')
        with self.assertRaises(DatasetIsNotACubeError) as cm:
            decode_cube(dataset, force_geographic=True)
        self.assertEqual("Grid mapping must use geographic CRS,"
                         " but was 'ETRS89 / UTM zone 32N'",
                         f'{cm.exception}')


class EncodeCubeTest(TestCase):
    def test_geographical_crs(self):
        cube = new_cube(variables=dict(a=1, b=2, c=3))
        gm = GridMapping.from_dataset(cube)

        dataset = encode_cube(cube, gm)
        self.assertIs(cube, dataset)

        dataset = encode_cube(cube, gm,
                              xr.Dataset(dict(d=True)))
        self.assertIsInstance(dataset, xr.Dataset)
        self.assertEqual({'a', 'b', 'c', 'd'}, set(dataset.data_vars))

    def test_non_geographical_crs(self):
        cube = new_cube(x_name='x',
                        y_name='y',
                        crs='epsg:25832',
                        variables=dict(a=1, b=2, c=3))
        gm = GridMapping.from_dataset(cube)
        dataset = encode_cube(cube,
                              gm,
                              xr.Dataset(dict(d=True)))
        self.assertIsInstance(dataset, xr.Dataset)
        self.assertEqual({'a', 'b', 'c', 'd', 'crs'}, set(dataset.data_vars))


class TestNormalize(TestCase):

    def test_normalize_zonal_lat_lon(self):
        resolution = 10
        lat_size = 3
        lat_coords = np.arange(0, 30, resolution)
        lon_coords = [i + 5. for i in np.arange(-180.0, 180.0, resolution)]
        lon_size = len(lon_coords)
        one_more_dim_size = 2
        one_more_dim_coords = np.random.random(2)

        var_values_1_1d = xr.DataArray(np.random.random(lat_size),
                                       coords=[('latitude_centers', lat_coords)],
                                       dims=['latitude_centers'],
                                       attrs=dict(chunk_sizes=[lat_size],
                                                  dimensions=['latitude_centers']))
        var_values_1_1d.encoding = {'chunks': (lat_size,)}
        var_values_1_2d = xr.DataArray(np.array([var_values_1_1d.values for _ in lon_coords]).T,
                                       coords={'lat': lat_coords, 'lon': lon_coords},
                                       dims=['lat', 'lon'],
                                       attrs=dict(chunk_sizes=[lat_size, lon_size],
                                                  dimensions=['lat', 'lon']))
        var_values_1_2d.encoding = {'chunks': (lat_size, lon_size)}
        var_values_2_2d = xr.DataArray(np.random.random(lat_size * one_more_dim_size).
                                       reshape(lat_size, one_more_dim_size),
                                       coords={'latitude_centers': lat_coords,
                                               'one_more_dim': one_more_dim_coords},
                                       dims=['latitude_centers', 'one_more_dim'],
                                       attrs=dict(chunk_sizes=[lat_size, one_more_dim_size],
                                                  dimensions=['latitude_centers', 'one_more_dim']))
        var_values_2_2d.encoding = {'chunks': (lat_size, one_more_dim_size)}
        var_values_2_3d = xr.DataArray(np.array([var_values_2_2d.values for _ in lon_coords]).T,
                                       coords={'one_more_dim': one_more_dim_coords,
                                               'lat': lat_coords,
                                               'lon': lon_coords, },
                                       dims=['one_more_dim', 'lat', 'lon', ],
                                       attrs=dict(chunk_sizes=[one_more_dim_size,
                                                               lat_size,
                                                               lon_size],
                                                  dimensions=['one_more_dim', 'lat', 'lon']))
        var_values_2_3d.encoding = {'chunks': (one_more_dim_size, lat_size, lon_size)}

        dataset = xr.Dataset({'first': var_values_1_1d, 'second': var_values_2_2d})
        expected = xr.Dataset({'first': var_values_1_2d, 'second': var_values_2_3d})
        expected = expected.assign_coords(
            lon_bnds=xr.DataArray([[i - (resolution / 2), i + (resolution / 2)] for i in expected.lon.values],
                                  dims=['lon', 'bnds']))
        expected = expected.assign_coords(
            lat_bnds=xr.DataArray([[i - (resolution / 2), i + (resolution / 2)] for i in expected.lat.values],
                                  dims=['lat', 'bnds']))
        actual = normalize_dataset(dataset)

        xr.testing.assert_equal(actual, expected)
        self.assertEqual(actual.first.chunk_sizes, expected.first.chunk_sizes)
        self.assertEqual(actual.second.chunk_sizes, expected.second.chunk_sizes)

    def test_normalize_lon_lat_2d(self):
        """
        Test nominal execution
        """
        dims = ('time', 'y', 'x')
        attrs = {'valid_min': 0., 'valid_max': 1.}

        t_size = 2
        y_size = 3
        x_size = 4

        a_data = np.random.random_sample((t_size, y_size, x_size))
        b_data = np.random.random_sample((t_size, y_size, x_size))
        time_data = [1, 2]
        lat_data = [[10., 10., 10., 10.],
                    [20., 20., 20., 20.],
                    [30., 30., 30., 30.]]
        lon_data = [[-10., 0., 10., 20.],
                    [-10., 0., 10., 20.],
                    [-10., 0., 10., 20.]]
        dataset = xr.Dataset({'a': (dims, a_data, attrs),
                              'b': (dims, b_data, attrs)
                              },
                             {'time': (('time',), time_data),
                              'lat': (('y', 'x'), lat_data),
                              'lon': (('y', 'x'), lon_data)
                              },
                             {'geospatial_lon_min': -15.,
                              'geospatial_lon_max': 25.,
                              'geospatial_lat_min': 5.,
                              'geospatial_lat_max': 35.
                              }
                             )

        new_dims = ('time', 'lat', 'lon')
        expected = xr.Dataset({'a': (new_dims, a_data, attrs),
                               'b': (new_dims, b_data, attrs)},
                              {'time': (('time',), time_data),
                               'lat': (('lat',), [10., 20., 30.]),
                               'lon': (('lon',), [-10., 0., 10., 20.]),
                               },
                              {'geospatial_lon_min': -15.,
                               'geospatial_lon_max': 25.,
                               'geospatial_lat_min': 5.,
                               'geospatial_lat_max': 35.})

        actual = normalize_dataset(dataset)
        xr.testing.assert_equal(actual, expected)

    def test_normalize_lon_lat(self):
        """
        Test nominal execution
        """
        dataset = xr.Dataset({'first': (['latitude',
                                         'longitude'], [[1, 2, 3],
                                                        [2, 3, 4]])})
        expected = xr.Dataset({'first': (['lat', 'lon'], [[1, 2, 3],
                                                          [2, 3, 4]])})
        actual = normalize_dataset(dataset)
        assertDatasetEqual(actual, expected)

        dataset = xr.Dataset({'first': (['lat', 'long'], [[1, 2, 3],
                                                          [2, 3, 4]])})
        expected = xr.Dataset({'first': (['lat', 'lon'], [[1, 2, 3],
                                                          [2, 3, 4]])})
        actual = normalize_dataset(dataset)
        assertDatasetEqual(actual, expected)

        dataset = xr.Dataset({'first': (['latitude',
                                         'spacetime'], [[1, 2, 3],
                                                        [2, 3, 4]])})
        expected = xr.Dataset({'first': (['lat', 'spacetime'], [[1, 2, 3],
                                                                [2, 3, 4]])})
        actual = normalize_dataset(dataset)
        assertDatasetEqual(actual, expected)

        dataset = xr.Dataset({'first': (['zef', 'spacetime'], [[1, 2, 3],
                                                               [2, 3, 4]])})
        expected = xr.Dataset({'first': (['zef', 'spacetime'], [[1, 2, 3],
                                                                [2, 3, 4]])})
        actual = normalize_dataset(dataset)
        assertDatasetEqual(actual, expected)

    def test_normalize_does_not_reorder_increasing_lat(self):
        first = np.zeros([3, 45, 90])
        first[0, :, :] = np.eye(45, 90)
        ds = xr.Dataset({
            'first': (['time', 'lat', 'lon'], first),
            'second': (['time', 'lat', 'lon'], np.zeros([3, 45, 90])),
            'lat': np.linspace(-88, 88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 4)]}).chunk(
            chunks={'time': 1})

        actual = normalize_dataset(ds)
        xr.testing.assert_equal(actual, ds)

    def test_normalize_with_missing_time_dim(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101',
                               'time_coverage_end': '20121231'})
        norm_ds = normalize_dataset(ds)
        self.assertIsNot(norm_ds, ds)
        self.assertEqual(len(norm_ds.coords), 4)
        self.assertIn('lon', norm_ds.coords)
        self.assertIn('lat', norm_ds.coords)
        self.assertIn('time', norm_ds.coords)
        self.assertIn('time_bnds', norm_ds.coords)

        self.assertEqual(norm_ds.first.shape, (1, 90, 180))
        self.assertEqual(norm_ds.second.shape, (1, 90, 180))
        self.assertEqual(norm_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2012-07-01T12:00:00')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][0], xr.DataArray(pd.to_datetime('2012-01-01')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][1], xr.DataArray(pd.to_datetime('2012-12-31')))

    def test_normalize_with_time_called_t(self):
        ds = xr.Dataset({'first': (['time', 'lat', 'lon'], np.zeros([4, 90, 180])),
                         'second': (['time', 'lat', 'lon'], np.zeros([4, 90, 180])),
                         't': ('time', np.array(['2005-07-02T00:00:00.000000000',
                                                 '2006-07-02T12:00:00.000000000',
                                                 '2007-07-03T00:00:00.000000000',
                                                 '2008-07-02T00:00:00.000000000'], dtype='datetime64[ns]'))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '2005-01-17',
                               'time_coverage_end': '2008-08-17'})
        norm_ds = normalize_dataset(ds)
        self.assertIsNot(norm_ds, ds)
        self.assertEqual(len(norm_ds.coords), 3)
        self.assertIn('lon', norm_ds.coords)
        self.assertIn('lat', norm_ds.coords)
        self.assertIn('time', norm_ds.coords)

        self.assertEqual(norm_ds.first.shape, (4, 90, 180))
        self.assertEqual(norm_ds.second.shape, (4, 90, 180))
        self.assertEqual(norm_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2005-07-02T00:00')))

    def test_normalize_julian_day(self):
        """
        Test Julian Day -> Datetime conversion
        """
        tuples = [gcal2jd(2000, x, 1) for x in range(1, 13)]

        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([88, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([88, 90, 12])),
            'lat': np.linspace(-88, 45, 88),
            'lon': np.linspace(-178, 178, 90),
            'time': [x[0] + x[1] for x in tuples]})
        ds.time.attrs['long_name'] = 'time in julian days'

        expected = xr.Dataset({
            'first': (['time', 'lat', 'lon'], np.zeros([12, 88, 90])),
            'second': (['time', 'lat', 'lon'], np.zeros([12, 88, 90])),
            'lat': np.linspace(-88, 45, 88),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})
        expected.time.attrs['long_name'] = 'time'

        actual = normalize_dataset(ds)

        assertDatasetEqual(actual, expected)


class AdjustSpatialTest(TestCase):
    def test_nominal(self):
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'lat': np.linspace(-88, 88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})

        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        ds1 = adjust_spatial_attrs(ds)

        # Make sure original dataset is not altered
        with self.assertRaises(KeyError):
            # noinspection PyStatementEffect
            ds.attrs['geospatial_lat_min']

        # Make sure expected values are in the new dataset
        self.assertEqual(ds1.attrs['geospatial_lat_min'], -90)
        self.assertEqual(ds1.attrs['geospatial_lat_max'], 90)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds1.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_lon_min'], -180)
        self.assertEqual(ds1.attrs['geospatial_lon_max'], 180)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((-180.0 -90.0, -180.0 90.0, 180.0 90.0,'
                         ' 180.0 -90.0, -180.0 -90.0))')

        # Test existing attributes update
        lon_min, lat_min, lon_max, lat_max = -20, -40, 60, 40
        indexers = {'lon': slice(lon_min, lon_max),
                    'lat': slice(lat_min, lat_max)}
        ds2 = ds1.sel(**indexers)
        ds2 = adjust_spatial_attrs(ds2)

        self.assertEqual(ds2.attrs['geospatial_lat_min'], -42)
        self.assertEqual(ds2.attrs['geospatial_lat_max'], 42)
        self.assertEqual(ds2.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds2.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_lon_min'], -20)
        self.assertEqual(ds2.attrs['geospatial_lon_max'], 60)
        self.assertEqual(ds2.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds2.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_bounds'],
                         'POLYGON((-20.0 -42.0, -20.0 42.0, 60.0 42.0, 60.0'
                         ' -42.0, -20.0 -42.0))')

    def test_nominal_inverted(self):
        # Inverted lat
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'lat': np.linspace(88, -88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})

        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        ds1 = adjust_spatial_attrs(ds)

        # Make sure original dataset is not altered
        with self.assertRaises(KeyError):
            # noinspection PyStatementEffect
            ds.attrs['geospatial_lat_min']

        # Make sure expected values are in the new dataset
        self.assertEqual(ds1.attrs['geospatial_lat_min'], -90)
        self.assertEqual(ds1.attrs['geospatial_lat_max'], 90)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds1.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_lon_min'], -180)
        self.assertEqual(ds1.attrs['geospatial_lon_max'], 180)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((-180.0 -90.0, -180.0 90.0, 180.0 90.0,'
                         ' 180.0 -90.0, -180.0 -90.0))')

        # Test existing attributes update
        lon_min, lat_min, lon_max, lat_max = -20, -40, 60, 40
        indexers = {'lon': slice(lon_min, lon_max),
                    'lat': slice(lat_max, lat_min)}
        ds2 = ds1.sel(**indexers)
        ds2 = adjust_spatial_attrs(ds2)

        self.assertEqual(ds2.attrs['geospatial_lat_min'], -42)
        self.assertEqual(ds2.attrs['geospatial_lat_max'], 42)
        self.assertEqual(ds2.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds2.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_lon_min'], -20)
        self.assertEqual(ds2.attrs['geospatial_lon_max'], 60)
        self.assertEqual(ds2.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds2.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_bounds'],
                         'POLYGON((-20.0 -42.0, -20.0 42.0, 60.0 42.0, 60.0'
                         ' -42.0, -20.0 -42.0))')

    def test_bnds(self):
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'lat': np.linspace(-88, 88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})

        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        lat_bnds = np.empty([len(ds.lat), 2])
        lon_bnds = np.empty([len(ds.lon), 2])
        ds['nv'] = [0, 1]

        lat_bnds[:, 0] = ds.lat.values - 2
        lat_bnds[:, 1] = ds.lat.values + 2
        lon_bnds[:, 0] = ds.lon.values - 2
        lon_bnds[:, 1] = ds.lon.values + 2

        ds['lat_bnds'] = (['lat', 'nv'], lat_bnds)
        ds['lon_bnds'] = (['lon', 'nv'], lon_bnds)

        ds.lat.attrs['bounds'] = 'lat_bnds'
        ds.lon.attrs['bounds'] = 'lon_bnds'

        ds1 = adjust_spatial_attrs(ds)

        # Make sure original dataset is not altered
        with self.assertRaises(KeyError):
            # noinspection PyStatementEffect
            ds.attrs['geospatial_lat_min']

        # Make sure expected values are in the new dataset
        self.assertEqual(ds1.attrs['geospatial_lat_min'], -90)
        self.assertEqual(ds1.attrs['geospatial_lat_max'], 90)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds1.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_lon_min'], -180)
        self.assertEqual(ds1.attrs['geospatial_lon_max'], 180)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((-180.0 -90.0, -180.0 90.0, 180.0 90.0,'
                         ' 180.0 -90.0, -180.0 -90.0))')

        # Test existing attributes update
        lon_min, lat_min, lon_max, lat_max = -20, -40, 60, 40
        indexers = {'lon': slice(lon_min, lon_max),
                    'lat': slice(lat_min, lat_max)}
        ds2 = ds1.sel(**indexers)
        ds2 = adjust_spatial_attrs(ds2)

        self.assertEqual(ds2.attrs['geospatial_lat_min'], -42)
        self.assertEqual(ds2.attrs['geospatial_lat_max'], 42)
        self.assertEqual(ds2.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds2.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_lon_min'], -20)
        self.assertEqual(ds2.attrs['geospatial_lon_max'], 60)
        self.assertEqual(ds2.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds2.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_bounds'],
                         'POLYGON((-20.0 -42.0, -20.0 42.0, 60.0 42.0, 60.0'
                         ' -42.0, -20.0 -42.0))')

    def test_bnds_inverted(self):
        # Inverted lat
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
            'lat': np.linspace(88, -88, 45),
            'lon': np.linspace(-178, 178, 90),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})

        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        lat_bnds = np.empty([len(ds.lat), 2])
        lon_bnds = np.empty([len(ds.lon), 2])
        ds['nv'] = [0, 1]

        lat_bnds[:, 0] = ds.lat.values + 2
        lat_bnds[:, 1] = ds.lat.values - 2
        lon_bnds[:, 0] = ds.lon.values - 2
        lon_bnds[:, 1] = ds.lon.values + 2

        ds['lat_bnds'] = (['lat', 'nv'], lat_bnds)
        ds['lon_bnds'] = (['lon', 'nv'], lon_bnds)

        ds.lat.attrs['bounds'] = 'lat_bnds'
        ds.lon.attrs['bounds'] = 'lon_bnds'

        ds1 = adjust_spatial_attrs(ds)

        # Make sure original dataset is not altered
        with self.assertRaises(KeyError):
            # noinspection PyStatementEffect
            ds.attrs['geospatial_lat_min']

        # Make sure expected values are in the new dataset
        self.assertEqual(ds1.attrs['geospatial_lat_min'], -90)
        self.assertEqual(ds1.attrs['geospatial_lat_max'], 90)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds1.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_lon_min'], -180)
        self.assertEqual(ds1.attrs['geospatial_lon_max'], 180)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((-180.0 -90.0, -180.0 90.0, 180.0 90.0,'
                         ' 180.0 -90.0, -180.0 -90.0))')

        # Test existing attributes update
        lon_min, lat_min, lon_max, lat_max = -20, -40, 60, 40
        indexers = {'lon': slice(lon_min, lon_max),
                    'lat': slice(lat_max, lat_min)}
        ds2 = ds1.sel(**indexers)
        ds2 = adjust_spatial_attrs(ds2)

        self.assertEqual(ds2.attrs['geospatial_lat_min'], -42)
        self.assertEqual(ds2.attrs['geospatial_lat_max'], 42)
        self.assertEqual(ds2.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertEqual(ds2.attrs['geospatial_lat_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_lon_min'], -20)
        self.assertEqual(ds2.attrs['geospatial_lon_max'], 60)
        self.assertEqual(ds2.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds2.attrs['geospatial_lon_resolution'], 4)
        self.assertEqual(ds2.attrs['geospatial_bounds'],
                         'POLYGON((-20.0 -42.0, -20.0 42.0, 60.0 42.0, 60.0 -42.0, -20.0 -42.0))')

    def test_once_cell_with_bnds(self):
        # Only one cell in lat/lon
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([1, 1, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([1, 1, 12])),
            'lat': np.array([52.5]),
            'lon': np.array([11.5]),
            'lat_bnds': (['lat', 'bnds'], np.array([[52.4, 52.6]])),
            'lon_bnds': (['lon', 'bnds'], np.array([[11.4, 11.6]])),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})
        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        ds1 = adjust_spatial_attrs(ds)
        self.assertAlmostEqual(ds1.attrs['geospatial_lat_resolution'], 0.2)
        self.assertAlmostEqual(ds1.attrs['geospatial_lat_min'], 52.4)
        self.assertAlmostEqual(ds1.attrs['geospatial_lat_max'], 52.6)
        self.assertEqual(ds1.attrs['geospatial_lat_units'], 'degrees_north')
        self.assertAlmostEqual(ds1.attrs['geospatial_lon_resolution'], 0.2)
        self.assertAlmostEqual(ds1.attrs['geospatial_lon_min'], 11.4)
        self.assertAlmostEqual(ds1.attrs['geospatial_lon_max'], 11.6)
        self.assertEqual(ds1.attrs['geospatial_lon_units'], 'degrees_east')
        self.assertEqual(ds1.attrs['geospatial_bounds'],
                         'POLYGON((11.4 52.4, 11.4 52.6, 11.6 52.6, 11.6 52.4, 11.4 52.4))')

    def test_once_cell_without_bnds(self):
        # Only one cell in lat/lon
        ds = xr.Dataset({
            'first': (['lat', 'lon', 'time'], np.zeros([1, 1, 12])),
            'second': (['lat', 'lon', 'time'], np.zeros([1, 1, 12])),
            'lat': np.array([52.5]),
            'lon': np.array([11.5]),
            'time': [datetime(2000, x, 1) for x in range(1, 13)]})
        ds.lon.attrs['units'] = 'degrees_east'
        ds.lat.attrs['units'] = 'degrees_north'

        ds2 = adjust_spatial_attrs(ds)
        # Datasets should be the same --> not modified
        self.assertIs(ds2, ds)


class NormalizeCoordVarsTest(TestCase):

    def test_ds_with_potential_coords(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180])),
                         'lat_bnds': (['lat', 'bnds'], np.zeros([90, 2])),
                         'lon_bnds': (['lon', 'bnds'], np.zeros([180, 2]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)})

        new_ds = normalize_coord_vars(ds)

        self.assertIsNot(ds, new_ds)
        self.assertEqual(len(new_ds.coords), 4)
        self.assertIn('lon', new_ds.coords)
        self.assertIn('lat', new_ds.coords)
        self.assertIn('lat_bnds', new_ds.coords)
        self.assertIn('lon_bnds', new_ds.coords)

        self.assertEqual(len(new_ds.data_vars), 2)
        self.assertIn('first', new_ds.data_vars)
        self.assertIn('second', new_ds.data_vars)

    def test_ds_with_potential_coords_and_bounds(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180])),
                         'lat_bnds': (['lat', 'bnds'], np.zeros([90, 2])),
                         'lon_bnds': (['lon', 'bnds'], np.zeros([180, 2])),
                         'lat': (['lat'], np.linspace(-89.5, 89.5, 90)),
                         'lon': (['lon'], np.linspace(-179.5, 179.5, 180))})

        new_ds = normalize_coord_vars(ds)

        self.assertIsNot(ds, new_ds)
        self.assertEqual(len(new_ds.coords), 4)
        self.assertIn('lon', new_ds.coords)
        self.assertIn('lat', new_ds.coords)
        self.assertIn('lat_bnds', new_ds.coords)
        self.assertIn('lon_bnds', new_ds.coords)

        self.assertEqual(len(new_ds.data_vars), 2)
        self.assertIn('first', new_ds.data_vars)
        self.assertIn('second', new_ds.data_vars)

    def test_ds_with_no_potential_coords(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101'})
        new_ds = normalize_coord_vars(ds)
        self.assertIs(ds, new_ds)


class NormalizeMissingTimeTest(TestCase):
    def test_ds_without_time(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101',
                               'time_coverage_end': '20121231'})

        new_ds = normalize_missing_time(ds)

        self.assertIsNot(ds, new_ds)
        self.assertEqual(len(new_ds.coords), 4)
        self.assertIn('lon', new_ds.coords)
        self.assertIn('lat', new_ds.coords)
        self.assertIn('time', new_ds.coords)
        self.assertIn('time_bnds', new_ds.coords)

        self.assertEqual(new_ds.coords['time'].attrs.get('long_name'), 'time')
        self.assertEqual(new_ds.coords['time'].attrs.get('bounds'), 'time_bnds')

        self.assertEqual(new_ds.first.shape, (1, 90, 180))
        self.assertEqual(new_ds.second.shape, (1, 90, 180))
        self.assertEqual(new_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2012-07-01T12:00:00')))
        self.assertEqual(new_ds.coords['time'].attrs.get('long_name'), 'time')
        self.assertEqual(new_ds.coords['time'].attrs.get('bounds'), 'time_bnds')
        self.assertEqual(new_ds.coords['time_bnds'][0][0], xr.DataArray(pd.to_datetime('2012-01-01')))
        self.assertEqual(new_ds.coords['time_bnds'][0][1], xr.DataArray(pd.to_datetime('2012-12-31')))
        self.assertEqual(new_ds.coords['time_bnds'].attrs.get('long_name'), 'time')

    def test_ds_without_bounds(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101'})

        new_ds = normalize_missing_time(ds)

        self.assertIsNot(ds, new_ds)
        self.assertEqual(len(new_ds.coords), 3)
        self.assertIn('lon', new_ds.coords)
        self.assertIn('lat', new_ds.coords)
        self.assertIn('time', new_ds.coords)
        self.assertNotIn('time_bnds', new_ds.coords)

        self.assertEqual(new_ds.first.shape, (1, 90, 180))
        self.assertEqual(new_ds.second.shape, (1, 90, 180))
        self.assertEqual(new_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2012-01-01')))
        self.assertEqual(new_ds.coords['time'].attrs.get('long_name'), 'time')
        self.assertEqual(new_ds.coords['time'].attrs.get('bounds'), None)

    def test_ds_without_time_attrs(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)})

        new_ds = normalize_missing_time(ds)
        self.assertIs(ds, new_ds)

    def test_ds_with_cftime(self):
        time_data = xr.cftime_range(start='2010-01-01T00:00:00',
                                    periods=6,
                                    freq='D',
                                    calendar='gregorian').values
        ds = xr.Dataset({'first': (['time', 'lat', 'lon'], np.zeros([6, 90, 180])),
                         'second': (['time', 'lat', 'lon'], np.zeros([6, 90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180),
                                'time': time_data},
                        attrs={'time_coverage_start': '20120101',
                               'time_coverage_end': '20121231'})
        new_ds = normalize_missing_time(ds)
        self.assertIs(ds, new_ds)

    def test_normalize_with_missing_time_dim(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        attrs={'time_coverage_start': '20120101',
                               'time_coverage_end': '20121231'})
        norm_ds = normalize_dataset(ds)
        self.assertIsNot(norm_ds, ds)
        self.assertEqual(len(norm_ds.coords), 4)
        self.assertIn('lon', norm_ds.coords)
        self.assertIn('lat', norm_ds.coords)
        self.assertIn('time', norm_ds.coords)
        self.assertIn('time_bnds', norm_ds.coords)

        self.assertEqual(norm_ds.first.shape, (1, 90, 180))
        self.assertEqual(norm_ds.second.shape, (1, 90, 180))
        self.assertEqual(norm_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2012-07-01T12:00:00')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][0], xr.DataArray(pd.to_datetime('2012-01-01')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][1], xr.DataArray(pd.to_datetime('2012-12-31')))

    def test_normalize_with_missing_time_dim_from_filename(self):
        ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),
                         'second': (['lat', 'lon'], np.zeros([90, 180]))},
                        coords={'lat': np.linspace(-89.5, 89.5, 90),
                                'lon': np.linspace(-179.5, 179.5, 180)},
                        )
        ds_encoding = dict(source='20150204_etfgz_20170309_dtsrgth')
        ds.encoding.update(ds_encoding)
        norm_ds = normalize_dataset(ds)
        self.assertIsNot(norm_ds, ds)
        self.assertEqual(len(norm_ds.coords), 4)
        self.assertIn('lon', norm_ds.coords)
        self.assertIn('lat', norm_ds.coords)
        self.assertIn('time', norm_ds.coords)
        self.assertIn('time_bnds', norm_ds.coords)

        self.assertEqual(norm_ds.first.shape, (1, 90, 180))
        self.assertEqual(norm_ds.second.shape, (1, 90, 180))
        self.assertEqual(norm_ds.coords['time'][0], xr.DataArray(pd.to_datetime('2016-02-21T00:00:00')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][0], xr.DataArray(pd.to_datetime('2015-02-04')))
        self.assertEqual(norm_ds.coords['time_bnds'][0][1], xr.DataArray(pd.to_datetime('2017-03-09')))


class Fix360Test(TestCase):

    def test_fix_360_lon(self):
        # The following simulates a strangely geo-coded soil moisture dataset we found
        lon_size = 360
        lat_size = 130
        time_size = 12
        ds = xr.Dataset({
            'first': (['time', 'lat', 'lon'],
                      np.random.random_sample([time_size, lat_size, lon_size])),
            'second': (['time', 'lat', 'lon'],
                       np.random.random_sample([time_size, lat_size,
                                                lon_size]))},
            coords={'lon': np.linspace(1., 360., lon_size),
                    'lat': np.linspace(-65., 64., lat_size),
                    'time': [datetime(2000, x, 1)
                             for x in range(1, time_size + 1)]},
            attrs=dict(geospatial_lon_min=0.,
                       geospatial_lon_max=360.,
                       geospatial_lat_min=-65.5,
                       geospatial_lat_max=+64.5,
                       geospatial_lon_resolution=1.,
                       geospatial_lat_resolution=1.))

        new_ds = normalize_dataset(ds)
        self.assertIsNot(ds, new_ds)
        self.assertEqual(ds.dims, new_ds.dims)
        self.assertEqual(ds.sizes, new_ds.sizes)
        assert_array_almost_equal(new_ds.lon, np.linspace(-179.5, 179.5, 360))
        assert_array_almost_equal(new_ds.lat, np.linspace(-65., 64., 130))
        assert_array_almost_equal(new_ds.first[..., :180], ds.first[..., 180:])
        assert_array_almost_equal(new_ds.first[..., 180:], ds.first[..., :180])
        assert_array_almost_equal(new_ds.second[..., :180],
                                  ds.second[..., 180:])
        assert_array_almost_equal(new_ds.second[..., 180:],
                                  ds.second[..., :180])
        self.assertEqual(-180., new_ds.attrs['geospatial_lon_min'])
        self.assertEqual(+180., new_ds.attrs['geospatial_lon_max'])
        self.assertEqual(-65.5, new_ds.attrs['geospatial_lat_min'])
        self.assertEqual(+64.5, new_ds.attrs['geospatial_lat_max'])
        self.assertEqual(1., new_ds.attrs['geospatial_lon_resolution'])
        self.assertEqual(1., new_ds.attrs['geospatial_lat_resolution'])


class NormalizeDimOrderTest(TestCase):
    """
    Test normalize_cci_sea_level operation
    """

    def test_no_change(self):
        """
        Test nominal operation
        """
        lon_size = 360
        lat_size = 130
        time_size = 12
        ds = xr.Dataset({
            'first': (['time', 'lat', 'lon'],
                      np.random.random_sample([time_size, lat_size, lon_size])),
            'second': (['time', 'lat', 'lon'],
                       np.random.random_sample([time_size, lat_size,
                                                lon_size]))},
            coords={'lon': np.linspace(-179.5, -179.5, lon_size),
                    'lat': np.linspace(-65., 64., lat_size),
                    'time': [datetime(2000, x, 1)
                             for x in range(1, time_size + 1)]})
        ds2 = normalize_dataset(ds)
        self.assertIs(ds2, ds)

    def test_nominal(self):
        """
        Test nominal operation
        """
        ds = self.new_cci_seal_level_ds()
        ds2 = normalize_dataset(ds)
        self.assertIsNot(ds2, ds)
        self.assertIn('ampl', ds2)
        self.assertIn('phase', ds2)
        self.assertIn('time', ds2.coords)
        self.assertIn('time_bnds', ds2.coords)
        self.assertNotIn('time_step', ds2.coords)
        self.assertEqual(['time', 'period', 'lat', 'lon'], list(ds2.ampl.dims))
        self.assertEqual(['time', 'period', 'lat', 'lon'], list(ds2.phase.dims))

    @staticmethod
    def new_cci_seal_level_ds():
        period_size = 2
        lon_size = 4
        lat_size = 2

        dataset = xr.Dataset(dict(ampl=(['lat', 'lon', 'period'], np.ones(shape=(lat_size, lon_size, period_size))),
                                  phase=(['lat', 'lon', 'period'], np.zeros(shape=(lat_size, lon_size, period_size)))),
                             coords=dict(lon=np.array([-135, -45., 45., 135.]), lat=np.array([-45., 45.]),
                                         time=pd.to_datetime(
                                             ['1993-01-15T00:00:00.000000000', '1993-02-15T00:00:00.000000000',
                                              '2015-11-15T00:00:00.000000000', '2015-12-15T00:00:00.000000000'])))

        dataset.coords['time'].encoding.update(units='days since 1950-01-01', dtype=np.dtype(np.float32))
        dataset.coords['time'].attrs.update(long_name='time', standard_name='time')

        dataset.attrs['time_coverage_start'] = '1993-01-01 00:00:00'
        dataset.attrs['time_coverage_end'] = '2015-12-31 23:59:59'

        return dataset


# test/core/test_optimize.py

import os
import os.path
import unittest
from typing import Set, Union, Sequence

from xcube.constants import FORMAT_NAME_ZARR
from xcube.core.chunk import chunk_dataset
from xcube.core.dsio import rimraf
from xcube.core.new import new_cube
from xcube.core.optimize import optimize_dataset

TEST_CUBE = chunk_dataset(new_cube(time_periods=3, variables=dict(A=0.5, B=-1.5)),
                          chunk_sizes=dict(time=1, lat=180, lon=360), format_name=FORMAT_NAME_ZARR)

INPUT_CUBE_PATH = 'test.zarr'
OUTPUT_CUBE_PATH = 'test_opt.zarr'
OUTPUT_CUBE_PATTERN = '{input}_opt.zarr'

INPUT_CUBE_FILE_SET = {
    '.zattrs', '.zgroup',  '.zmetadata',
    'A/.zarray', 'A/.zattrs', 'A/0.0.0', 'A/1.0.0', 'A/2.0.0',
    'B/.zarray', 'B/.zattrs', 'B/0.0.0', 'B/1.0.0', 'B/2.0.0',
    'lat/.zarray', 'lat/.zattrs', 'lat/0',
    'lat_bnds/.zarray', 'lat_bnds/.zattrs', 'lat_bnds/0.0',
    'lon/.zarray', 'lon/.zattrs', 'lon/0',
    'lon_bnds/.zarray', 'lon_bnds/.zattrs', 'lon_bnds/0.0',
    'time/.zarray', 'time/.zattrs', 'time/0', 'time/1', 'time/2',
    'time_bnds/.zarray', 'time_bnds/.zattrs', 'time_bnds/0.0', 'time_bnds/1.0', 'time_bnds/2.0'
}


class OptimizeDatasetTest(unittest.TestCase):
    def _clear_outputs(self):
        rimraf(INPUT_CUBE_PATH)
        rimraf(OUTPUT_CUBE_PATH)

    def setUp(self):
        self._clear_outputs()
        TEST_CUBE.to_zarr(INPUT_CUBE_PATH)
        self.assertEqual(INPUT_CUBE_FILE_SET, list_file_set(INPUT_CUBE_PATH))

    def tearDown(self):
        self._clear_outputs()

    def test_optimize_dataset(self):
        self._test_optimize_dataset(unchunk_coords=False, in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=False,
                                    expected_cons_time_bnds=False)

    def test_optimize_dataset_in_place(self):
        self._test_optimize_dataset(unchunk_coords=False, in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=False,
                                    expected_cons_time_bnds=False)

    def test_optimize_dataset_unchunk_coords_true(self):
        self._test_optimize_dataset(unchunk_coords=True, in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_true_in_place(self):
        self._test_optimize_dataset(unchunk_coords=True, in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_str(self):
        self._test_optimize_dataset(unchunk_coords='time', in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=False)

    def test_optimize_dataset_unchunk_coords_str_in_place(self):
        self._test_optimize_dataset(unchunk_coords='time', in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=False)

    def test_optimize_dataset_unchunk_coords_str_list(self):
        self._test_optimize_dataset(unchunk_coords=['time_bnds'], in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=False,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_str_list_in_place(self):
        self._test_optimize_dataset(unchunk_coords=['time_bnds'], in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=False,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_str_tuple(self):
        self._test_optimize_dataset(unchunk_coords=('time_bnds', 'time'), in_place=False,
                                    expected_output_path=OUTPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=True)

    def test_optimize_dataset_unchunk_coords_str_tuple_in_place(self):
        self._test_optimize_dataset(unchunk_coords=('time_bnds', 'time'), in_place=True,
                                    expected_output_path=INPUT_CUBE_PATH,
                                    expected_cons_time=True,
                                    expected_cons_time_bnds=True)

    def _test_optimize_dataset(self, unchunk_coords: Union[bool, str, Sequence[str]], in_place: bool,
                               expected_output_path: str, expected_cons_time: bool = False,
                               expected_cons_time_bnds: bool = False):
        if not in_place:
            optimize_dataset(INPUT_CUBE_PATH, output_path=OUTPUT_CUBE_PATTERN,
                             in_place=in_place, unchunk_coords=unchunk_coords)
        else:
            optimize_dataset(INPUT_CUBE_PATH,
                             in_place=in_place, unchunk_coords=unchunk_coords)
        self._assert_consolidated(expected_output_path, expected_cons_time, expected_cons_time_bnds)

    def _assert_consolidated(self,
                             cube_path: str,
                             cons_time: bool = False,
                             cons_time_bnds: bool = False):
        self.assertTrue(os.path.isdir(cube_path))
        expected_files = set(INPUT_CUBE_FILE_SET)
        expected_files.add('.zmetadata')
        if cons_time:
            expected_files.remove('time/1')
            expected_files.remove('time/2')
        if cons_time_bnds:
            expected_files.remove('time_bnds/1.0')
            expected_files.remove('time_bnds/2.0')
        self.assertEqual(expected_files, list_file_set(cube_path))

    def test_failures(self):
        with self.assertRaises(RuntimeError) as cm:
            optimize_dataset('pippo', in_place=True, exception_type=RuntimeError)
        self.assertEqual('Input path must point to ZARR dataset directory.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            optimize_dataset(INPUT_CUBE_PATH, exception_type=RuntimeError)
        self.assertEqual('Output path must be given.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            optimize_dataset(INPUT_CUBE_PATH, output_path=INPUT_CUBE_PATH, exception_type=RuntimeError)
        self.assertEqual('Output path already exists.', f'{cm.exception}')

        with self.assertRaises(RuntimeError) as cm:
            optimize_dataset(INPUT_CUBE_PATH, output_path='./' + INPUT_CUBE_PATH, exception_type=RuntimeError)
        self.assertEqual('Output path already exists.', f'{cm.exception}')


def list_file_set(dir_path: str) -> Set[str]:
    """ Get set of all files in a directory and all of its sub-directories. """
    actual_files = set()
    for root, dirs, files in os.walk(dir_path):
        for f in files:
            rel_root = root[len(dir_path) + 1:]
            if rel_root:
                actual_files.add(rel_root + '/' + f)
            else:
                actual_files.add(f)
    return actual_files


# test/core/test_reproject.py

import unittest

import numpy as np
# noinspection PyUnresolvedReferences
from matplotlib import pyplot as plt
from numpy.testing import assert_array_almost_equal

from test.sampledata import create_highroc_dataset
from xcube.core.reproject import reproject_xy_to_wgs84

nan = np.nan


class ReprojectTest(unittest.TestCase):

    def test_reproject_xy_to_wgs84_highroc(self):
        dst_width = 12
        dst_height = 9

        dataset = create_highroc_dataset()
        proj_dataset = reproject_xy_to_wgs84(dataset,
                                             src_xy_var_names=('lon', 'lat'),
                                             src_xy_tp_var_names=('TP_longitude', 'TP_latitude'),
                                             src_xy_gcp_step=1,
                                             src_xy_tp_gcp_step=1,
                                             dst_size=(dst_width, dst_height))

        self.assertIsNotNone(proj_dataset)
        self.assertEqual(dict(lon=dst_width, lat=dst_height, bnds=2),
                         proj_dataset.sizes)

        self.assertIn('lon', proj_dataset)
        self.assertEqual(proj_dataset.lon.shape, (dst_width,))
        self.assertIn('lat', proj_dataset)
        self.assertEqual(proj_dataset.lat.shape, (dst_height,))

        self.assertIn('lon_bnds', proj_dataset)
        self.assertEqual(proj_dataset.lon_bnds.shape, (dst_width, 2))
        self.assertIn('lat_bnds', proj_dataset)
        self.assertEqual(proj_dataset.lat_bnds.shape, (dst_height, 2))

        expected_conc_chl = np.array([[7., 7., 11., 11., 11., 11., nan, nan, nan, nan, 5., 5.],
                                      [7., 7., 11., 11., 11., 11., nan, nan, nan, 21., 21., 21.],
                                      [5., 5., 10., 10., 10., 10., 2., 2., 2., 21., 21., 21.],
                                      [5., 5., 10., 10., 10., 2., 2., 2., 2., 21., 17., 17.],
                                      [5., 5., 10., 10., 10., 20., 20., 20., 20., 17., 17., 17.],
                                      [5., 16., 6., 6., 6., 20., 20., 20., 17., 17., nan, nan],
                                      [16., 16., 6., 6., 6., 20., nan, nan, nan, nan, nan, nan],
                                      [16., 16., 6., nan, nan, nan, nan, nan, nan, nan, nan, nan],
                                      [nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]],
                                     dtype=np.float64)
        self.assertIn('conc_chl', proj_dataset)
        # print(proj_dataset.conc_chl)
        self.assertEqual(proj_dataset.conc_chl.shape, (dst_height, dst_width))
        self.assertEqual(proj_dataset.conc_chl.dtype, np.float64)
        assert_array_almost_equal(proj_dataset.conc_chl, expected_conc_chl)

        expected_c2rcc_flags = np.array([[1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],
                                         [1., 1., 1., 1., 1., 1., 1., 1., 1., 2., 2., 2.],
                                         [1., 1., 4., 4., 4., 4., 1., 1., 1., 2., 2., 2.],
                                         [1., 1., 4., 4., 4., 1., 1., 1., 1., 2., 1., 1.],
                                         [1., 1., 4., 4., 4., 1., 1., 1., 1., 1., 1., 1.],
                                         [1., 8., 1., 1., 1., 1., 1., 1., 1., 1., nan, nan],
                                         [8., 8., 1., 1., 1., 1., nan, nan, nan, nan, nan, nan],
                                         [8., 8., 1., nan, nan, nan, nan, nan, nan, nan, nan, nan],
                                         [nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]],
                                        dtype=np.float64)
        self.assertIn('c2rcc_flags', proj_dataset)
        # print(proj_dataset.c2rcc_flags)
        self.assertEqual(proj_dataset.c2rcc_flags.shape, (dst_height, dst_width))
        self.assertEqual(proj_dataset.c2rcc_flags.dtype, np.float64)
        assert_array_almost_equal(proj_dataset.c2rcc_flags, expected_c2rcc_flags)


# test/core/test_schema.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.new import new_cube
from xcube.core.schema import CubeSchema, get_dataset_chunks


class CubeSchemaTest(unittest.TestCase):

    def test_new(self):
        cube = new_cube(variables=dict(a=2, b=3, c=4))
        schema = CubeSchema.new(cube)
        self._assert_schema(schema, expected_shape=cube.a.shape)

    def test_new_chunked(self):
        cube = new_cube(variables=dict(a=2, b=3, c=4))
        cube = cube.chunk(dict(time=1, lat=90, lon=90))
        schema = CubeSchema.new(cube)
        self._assert_schema(schema, expected_shape=cube.a.shape, expected_chunks=(1, 90, 90))

    def test_xarray_accessor(self):
        # noinspection PyUnresolvedReferences
        import xcube.core.xarray
        cube = new_cube(variables=dict(a=2, b=3, c=4))
        schema = cube.xcube.schema
        self._assert_schema(schema, expected_shape=cube.a.shape)

    def _assert_schema(self, schema: CubeSchema, expected_shape=None, expected_chunks=None):
        self.assertEqual(3, schema.ndim)
        self.assertEqual(expected_shape, schema.shape)
        self.assertEqual(expected_chunks, schema.chunks)
        self.assertIn('time', schema.coords)
        self.assertIn('lat', schema.coords)
        self.assertIn('lon', schema.coords)
        self.assertEqual(('time', 'lat', 'lon'), schema.dims)
        self.assertEqual('time', schema.time_dim)
        self.assertEqual('lon', schema.x_name)
        self.assertEqual('lat', schema.y_name)
        self.assertEqual('time', schema.time_name)
        self.assertEqual('lon', schema.x_dim)
        self.assertEqual('lat', schema.y_dim)
        self.assertEqual('time', schema.time_dim)
        self.assertEqual(expected_shape[-1], schema.x_size)
        self.assertEqual(expected_shape[-2], schema.y_size)
        self.assertEqual(expected_shape[0], schema.time_size)
        self.assertEqual(expected_shape[-1], schema.x_var.size)
        self.assertEqual(expected_shape[-2], schema.y_var.size)
        self.assertEqual(expected_shape[0], schema.time_var.size)

    def test_repr_html(self):
        cube = new_cube(variables=dict(a=2, b=3, c=4))
        cube = cube.chunk(dict(time=1, lat=90, lon=90))
        schema = CubeSchema.new(cube)
        self.assertEqual("<table>"
                         "<tr><td>Shape:</td><td>(5, 180, 360)</td></tr>"
                         "<tr><td>Chunk sizes:</td><td>(1, 90, 90)</td></tr>"
                         "<tr><td>Dimensions:</td><td>('time', 'lat', 'lon')</td></tr>"
                         "</table>",
                         schema._repr_html_())

    def test_constructor_with_invalid_args(self):
        cube = new_cube(variables=dict(t=273))
        schema = CubeSchema.new(cube)
        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(None, schema.coords)
        self.assertEqual('shape must be a sequence of integer sizes',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(schema.shape, None)
        self.assertEqual('coords must be a mapping from dimension names to label arrays',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(schema.shape, cube.coords, x_name=None)
        self.assertEqual('x_name must be given',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(schema.shape, cube.coords, y_name=None)
        self.assertEqual('y_name must be given',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            CubeSchema(schema.shape, cube.coords, time_name=None)
        self.assertEqual('time_name must be given',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape[1:], schema.coords)
        self.assertEqual('shape must have at least three dimensions',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape, schema.coords, dims=('lat', 'lon'))
        self.assertEqual('dims must have same length as shape',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape, schema.coords, dims=('lat', 'lon', 'time'))
        self.assertEqual("the first dimension in dims must be 'time'",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape, schema.coords, dims=('time', 'lon', 'lat'))
        self.assertEqual("the last two dimensions in dims must be 'lat' and 'lon'",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            CubeSchema(schema.shape, schema.coords, dims=schema.dims, chunks=(90, 90))
        self.assertEqual("chunks must have same length as shape",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            coords = dict(schema.coords)
            del coords['lat']
            CubeSchema(schema.shape, coords, dims=schema.dims, chunks=(1, 90, 90))
        self.assertEqual("missing variables 'lon', 'lat', 'time' in coords",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            coords = dict(schema.coords)
            lat = coords['lat']
            coords['lat'] = xr.DataArray(lat.values.reshape((1, len(lat))), dims=('b', lat.dims[0]), attrs=lat.attrs)
            CubeSchema(schema.shape, coords, dims=schema.dims, chunks=(1, 90, 90))
        self.assertEqual("variables 'lon', 'lat', 'time' in coords must be 1-D",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            coords = dict(schema.coords)
            lat = coords['lat']
            coords['lat'] = xr.DataArray(lat.values[1:], dims=('lat',), attrs=lat.attrs)
            CubeSchema(schema.shape, coords, dims=schema.dims, chunks=(1, 90, 90))
        self.assertEqual("number of labels of 'lat' in coords does not match shape",
                         f'{cm.exception}')

    def test_new_with_cube(self):
        cube = new_cube()
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("cube is empty",
                         f'{cm.exception}')

        cube = new_cube()
        del cube.coords['lon']
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("cube has no valid spatial coordinate variables",
                         f'{cm.exception}')

        cube = new_cube()
        del cube.coords['time']
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("cube has no valid time coordinate variable",
                         f'{cm.exception}')

        cube = new_cube(variables=dict(a=1, b=2))
        cube['c'] = xr.DataArray(np.array([1, 2, 3, 4, 5]), dims=('q',))
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("all variables must have same dimensions, but variable 'c' has dimensions ('q',)",
                         f'{cm.exception}')

        cube = new_cube(variables=dict(a=1, b=2))
        cube = cube.chunk(dict(time=1, lat=90, lon=90))
        cube['b'] = cube['b'].chunk(dict(time=1, lat=45, lon=90))
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("all variables must have same chunks, but variable 'b' has chunks (1, 45, 90)",
                         f'{cm.exception}')

        cube = new_cube(variables=dict(a=1, b=2))
        cube = cube.chunk(dict(time=1, lat=(44, 43, 46, 47), lon=90))
        with self.assertRaises(ValueError) as cm:
            CubeSchema.new(cube)
        self.assertEqual("dimension 'lat' of variable 'a' has chunks of different sizes: (44, 43, 46, 47)",
                         f'{cm.exception}')


class GetDatasetChunksTest(unittest.TestCase):

    def test_empty_dataset(self):
        dataset = xr.Dataset()
        self.assertEqual({}, get_dataset_chunks(dataset))

    def test_only_coords_dataset(self):
        dataset = xr.Dataset(
            coords=dict(
                x=xr.DataArray(np.linspace(0, 1, 100), dims='x').chunk(10),
                y=xr.DataArray(np.linspace(0, 1, 100), dims='y').chunk(20),
            )
        )
        self.assertEqual({}, get_dataset_chunks(dataset))

    def test_different_chunks(self):
        dataset = xr.Dataset(data_vars=dict(
            a=xr.DataArray(
                np.linspace(0, 1, 100 * 100).reshape((100, 100)),
                dims=('y', 'x')
            ).chunk((30, 10)),
            b=xr.DataArray(
                np.linspace(0, 1, 100 * 100).reshape((100, 100)),
                dims=('y', 'x')
            ).chunk((25, 15)),
            c=xr.DataArray(
                np.linspace(0, 1, 100 * 100).reshape((1, 100, 100)),
                dims=('time', 'y', 'x')
            ).chunk((1, 25, 10)),
            d=xr.DataArray(  # d is not chunked!
                np.linspace(0, 1, 100 * 100).reshape((1, 100, 100)),
                dims=('time', 'y', 'x')
            ),
        ))
        self.assertEqual(
            {'time': 1, 'x': 10, 'y': 25},
            get_dataset_chunks(dataset)
        )


# test/core/test_select.py

import itertools
import json
import math
import unittest
from collections.abc import MutableMapping
from typing import Dict, KeysView, Iterator, Sequence, Any

import cftime
import numpy as np
import xarray as xr

from test.sampledata import create_highroc_dataset
from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.select import select_spatial_subset
from xcube.core.select import select_subset
from xcube.core.select import select_temporal_subset
from xcube.core.select import select_variables_subset


class SelectVariablesSubsetTest(unittest.TestCase):
    def test_select_variables_subset_all(self):
        ds1 = create_highroc_dataset()
        # noinspection PyTypeChecker
        ds2 = select_variables_subset(ds1, None)
        self.assertIs(ds2, ds1)
        ds2 = select_variables_subset(ds1, ds1.data_vars.keys())
        self.assertIs(ds2, ds1)

    def test_select_variables_subset_none(self):
        ds1 = create_highroc_dataset()
        ds2 = select_variables_subset(ds1, [])
        self.assertEqual(0, len(ds2.data_vars))
        ds2 = select_variables_subset(ds1, ['bibo'])
        self.assertEqual(0, len(ds2.data_vars))

    def test_select_variables_subset_some(self):
        ds1 = create_highroc_dataset()
        self.assertEqual(36, len(ds1.data_vars))
        ds2 = select_variables_subset(ds1, ['conc_chl', 'c2rcc_flags', 'rtoa_10'])
        self.assertEqual(3, len(ds2.data_vars))


class SelectSpatialSubsetTest(unittest.TestCase):
    def test_select_spatial_subset_all_ij_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, ij_bbox=(0, 0, 4, 3))
        self.assertIs(ds2, ds1)

    def test_select_spatial_subset_some_ij_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, ij_bbox=(1, 1, 4, 3))
        self.assertEqual((2, 3), ds2.conc_chl.shape)

    def test_select_spatial_subset_none_ij_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, ij_bbox=(5, 6, 7, 8))
        self.assertEqual(None, ds2)
        ds2 = select_spatial_subset(ds1, ij_bbox=(-6, -4, 2, 2))
        self.assertEqual(None, ds2)

    def test_select_spatial_subset_all_xy_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, xy_bbox=(7.9, 53.9, 12., 56.4))
        self.assertIs(ds2, ds1)

    def test_select_spatial_subset_some_xy_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, xy_bbox=(8., 55, 10., 56.))
        self.assertEqual((3, 3), ds2.conc_chl.shape)

    def test_select_spatial_subset_none_xy_bbox(self):
        ds1 = create_highroc_dataset()
        ds2 = select_spatial_subset(ds1, xy_bbox=(13., 57., 15., 60.))
        self.assertEqual(None, ds2)
        ds2 = select_spatial_subset(ds1, xy_bbox=(5.5, 55, 6.5, 56))
        self.assertEqual(None, ds2)

    def test_select_spatial_subset_invalid_params(self):
        ds1 = create_highroc_dataset()
        with self.assertRaises(ValueError) as cm:
            select_spatial_subset(ds1, ij_bbox=(5, 6, 7, 8), xy_bbox=(0., 0., 1., 2.))
        self.assertEqual("Only one of ij_bbox and xy_bbox can be given", f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            select_spatial_subset(ds1)
        self.assertEqual("One of ij_bbox and xy_bbox must be given", f'{cm.exception}')

    def test_select_spatial_subset_descending_y_param(self):
        ds1 = new_cube(inverse_y=True)
        ds2 = select_spatial_subset(ds1, xy_bbox=(40., 40., 42., 42.))
        self.assertEqual(((2,), (2,)), (ds2.lon.shape, ds2.lat.shape))

    def test_select_spatial_subset_with_gm(self):
        ds1 = new_cube(inverse_y=True)
        ds2 = select_spatial_subset(ds1, xy_bbox=(40., 40., 42., 42.),
                                    grid_mapping=GridMapping.from_dataset(ds1))
        self.assertEqual(((2,), (2,)), (ds2.lon.shape, ds2.lat.shape))
        ds2 = select_spatial_subset(ds1, xy_bbox=(40., 40., 42., 42.),
                                    geo_coding=GridMapping.from_dataset(ds1))
        self.assertEqual(((2,), (2,)), (ds2.lon.shape, ds2.lat.shape))


class SelectTemporalSubsetTest(unittest.TestCase):
    def test_invalid_dataset(self):
        ds1 = xr.Dataset(dict(CHL=xr.DataArray([[1, 2], [2, 3]], dims=('lat', 'lon'))))
        with self.assertRaises(ValueError) as cm:
            select_temporal_subset(ds1, time_range=('2010-01-02', '2010-01-04'))
        self.assertEqual('cannot compute temporal subset: variable "time" not found in dataset',
                         f'{cm.exception}')

    def test_no_subset_for_open_interval(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8))
        ds2 = select_temporal_subset(ds1, time_range=(None, None))
        self.assertIs(ds2, ds1)

    def test_subset_for_closed_interval(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8))
        ds2 = select_temporal_subset(ds1, time_range=('2010-01-02', '2010-01-04'))
        self.assertIsNot(ds2, ds1)
        np.testing.assert_equal(np.array(['2010-01-02T12:00:00.000000000',
                                          '2010-01-03T12:00:00.000000000',
                                          '2010-01-04T12:00:00.000000000'],
                                         dtype='datetime64[ns]'),
                                ds2.time.values)

    def test_subset_for_upward_open_interval(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8))
        ds2 = select_temporal_subset(ds1, time_range=('2010-01-03', None))
        self.assertIsNot(ds2, ds1)
        np.testing.assert_equal(np.array(['2010-01-03T12:00:00.000000000',
                                          '2010-01-04T12:00:00.000000000',
                                          '2010-01-05T12:00:00.000000000'],
                                         dtype='datetime64[ns]'),
                                ds2.time.values)

    def test_subset_for_downward_open_interval(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8))
        ds2 = select_temporal_subset(ds1, time_range=(None, '2010-01-03'))
        self.assertIsNot(ds2, ds1)
        np.testing.assert_equal(np.array(['2010-01-01T12:00:00.000000000',
                                          '2010-01-02T12:00:00.000000000',
                                          '2010-01-03T12:00:00.000000000'],
                                         dtype='datetime64[ns]'),
                                ds2.time.values)

    def test_cf_time_subset(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8),
                       use_cftime=True,
                       time_dtype=None,
                       time_units='days since 1950-01-01',
                       time_calendar='julian')
        ds2 = select_temporal_subset(ds1, time_range=('2010-01-02', '2010-01-04'))
        self.assertIsNot(ds2, ds1)
        np.testing.assert_equal(np.array([cftime.DatetimeJulian(2010, 1, 2, 12, 0, 0),
                                          cftime.DatetimeJulian(2010, 1, 3, 12, 0, 0),
                                          cftime.DatetimeJulian(2010, 1, 4, 12, 0, 0)],
                                         dtype='object'),
                                ds2.time.values)


class SelectSubsetTest(unittest.TestCase):
    def test_all_params(self):
        ds1 = new_cube(variables=dict(analysed_sst=0.6, mask=8), drop_bounds=True)
        ds2 = select_subset(ds1,
                            var_names=['analysed_sst'],
                            bbox=(10, 50, 15, 55),
                            time_range=('2010-01-02', '2010-01-04'))

        self.assertEqual({'analysed_sst'}, set(ds2.data_vars))
        self.assertEqual({'time', 'lat', 'lon'}, set(ds2.coords))

        np.testing.assert_almost_equal(np.array([10.5, 11.5, 12.5, 13.5, 14.5]),
                                       ds2.lon.values)

        np.testing.assert_almost_equal(np.array([50.5, 51.5, 52.5, 53.5, 54.5]),
                                       ds2.lat.values)

        np.testing.assert_equal(np.array(['2010-01-02T12:00:00.000000000',
                                          '2010-01-03T12:00:00.000000000',
                                          '2010-01-04T12:00:00.000000000'],
                                         dtype='datetime64[ns]'),
                                ds2.time.values)

    def test_xy_bbox_with_large_dataset(self):
        ds = self._new_large_dataset()
        ds_subset = select_spatial_subset(ds, xy_bbox=(0., 0., 5.0, 2.5))
        self._assert_large_dataset_subset(ds_subset)

    def test_ij_bbox_with_large_dataset(self):
        ds = self._new_large_dataset()
        ds_subset = select_spatial_subset(ds, ij_bbox=(129600 // 2, 64800 // 2, 129600 // 2 + 1800, 64800 // 2 + 900))
        self._assert_large_dataset_subset(ds_subset)

    def _new_large_dataset(self):
        ds = new_virtual_dataset(lon_size=129600, lat_size=64800, time_size=5,
                                 lon_chunk=1296, lat_chunk=648, time_chunk=1)
        # print(ds)
        self.assertIsInstance(ds, xr.Dataset)
        self.assertIn('conc_chl', ds)
        self.assertEqual(('time', 'lat', 'lon'), ds.conc_chl.dims)
        self.assertEqual((5, 64800, 129600), ds.conc_chl.shape)
        self.assertEqual((5 * (1,), 100 * (648,), 100 * (1296,)), ds.conc_chl.chunks)
        return ds

    def _assert_large_dataset_subset(self, ds_subset):
        # print(ds_subset)
        self.assertIsInstance(ds_subset, xr.Dataset)
        self.assertIn('conc_chl', ds_subset)
        self.assertEqual(('time', 'lat', 'lon'), ds_subset.conc_chl.dims)
        self.assertEqual((5, 900, 1800), ds_subset.conc_chl.shape)
        self.assertEqual(((1, 1, 1, 1, 1), (648, 252), (1296, 504)), ds_subset.conc_chl.chunks)


def new_virtual_dataset(lon_size: int = 360,
                        lat_size: int = 180,
                        time_size: int = 5,
                        lon_chunk: int = None,
                        lat_chunk: int = None,
                        time_chunk: int = None):
    lon_res = 360. / lon_size
    lat_res = 180. / lat_size

    lon_values = np.linspace(-180. + lon_res / 2, 180. - lon_res / 2, lon_size)
    lat_values = np.linspace(-90. + lat_res / 2, 90. - lat_res / 2, lat_size)
    day1 = 365 * (2021 - 1970)
    time_values = np.linspace(day1, day1 + time_size, time_size, dtype='int64')

    chunk_store = VirtualChunkStore()
    chunk_store.add_unchunked_array('lon', ['lon'], lon_values)
    chunk_store.add_unchunked_array('lat', ['lat'], lat_values)
    chunk_store.add_unchunked_array('time', ['time'], time_values, attrs={
        "calendar": "gregorian",
        "units": "days since 1970-01-01"
    })
    chunk_store.add_chunked_array('conc_chl',
                                  ['time', 'lat', 'lon'],
                                  [time_size, lat_size, lon_size],
                                  [time_chunk or time_size, lat_chunk or lat_size, lon_chunk or lon_size])

    return xr.open_zarr(chunk_store)


class VirtualChunkStore(MutableMapping):

    def __init__(self, entries: Dict[str, bytes] = None):
        if entries:
            self._entries: Dict[str, bytes] = dict(entries)
        else:
            self._entries: Dict[str, bytes] = {
                ".zgroup": bytes(json.dumps({"zarr_format": 2}, indent=2), encoding='utf-8'),
                ".zattrs": bytes(json.dumps({}, indent=2), encoding='utf-8'),
            }

    def add_unchunked_array(self,
                            name: str,
                            dims: Sequence[str],
                            values: np.ndarray,
                            attrs: Dict[str, Any] = None):
        self._entries.update(self.get_array_entries_unchunked(name, dims, values, attrs=attrs))

    def add_chunked_array(self,
                          name: str,
                          dims: Sequence[str],
                          shape: Sequence[int],
                          chunks: Sequence[int],
                          value: float = None):
        self._entries.update(self.get_array_entries_chunked(name, dims, shape, chunks, value=value))

    def keys(self) -> KeysView[str]:
        return self._entries.keys()

    def __iter__(self) -> Iterator[str]:
        return iter(self._entries.keys())

    def __len__(self) -> int:
        return len(self._entries)

    def __contains__(self, key) -> bool:
        return key in self._entries

    def __getitem__(self, key: str) -> bytes:
        return self._entries[key]

    def __setitem__(self, key: str, value: bytes) -> None:
        self._entries[key] = value

    def __delitem__(self, key: str) -> None:
        del self._entries[key]

    @classmethod
    def get_array_entries_unchunked(cls,
                                    name: str,
                                    dims: Sequence[str],
                                    values: np.ndarray,
                                    attrs: Dict[str, Any] = None) -> Dict[str, bytes]:

        zarray = {
            "zarr_format": 2,
            "chunks": list(values.shape),
            "shape": list(values.shape),
            "dtype": str(values.dtype.str),
            "order": "C",
            "fill_value": None,
            "compressor": None,
            "filters": None,
        }

        zattrs = {
            # For xarray
            "_ARRAY_DIMENSIONS": dims,
        }
        if attrs:
            zattrs.update(attrs)

        return {
            name + '/.zarray': bytes(json.dumps(zarray, indent=2), encoding='utf-8'),
            name + '/.zattrs': bytes(json.dumps(zattrs, indent=2), encoding='utf-8'),
            name + '/0': values.tobytes(order="C")
        }

    @classmethod
    def get_array_entries_chunked(cls,
                                  name: str,
                                  dims: Sequence[str],
                                  shape: Sequence[int],
                                  chunks: Sequence[int],
                                  value: float = None) -> Dict[str, bytes]:
        dtype = np.dtype('float64')

        zarray = {
            "zarr_format": 2,
            "dtype": str(dtype.str),
            "shape": list(shape),
            "chunks": list(chunks),
            "order": "C",
            "fill_value": "NaN",
            "compressor": None,
            "filters": None,
        }

        zattrs = {
            # For xarray
            "_ARRAY_DIMENSIONS": dims
        }

        entries = {
            name + '/.zarray': bytes(json.dumps(zarray, indent=2), encoding='utf-8'),
            name + '/.zattrs': bytes(json.dumps(zattrs, indent=2), encoding='utf-8'),
        }

        if value is not None:
            values = np.full(shape, value, dtype=dtype)
            data = values.tobytes(order="C")
            chunk_indices = [tuple(range(math.ceil(s / c))) for s, c in zip(shape, chunks)]
            for index in itertools.product(*chunk_indices):
                key = f'{name}/{"/".join(index)}'
                entries[key] = data

        return entries


# test/core/test_subsampling.py

#  The MIT License (MIT)
#  Copyright (c) 2022 by the xcube development team and contributors
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.

import unittest

import numpy as np
import xarray as xr

from xcube.core.new import new_cube
from xcube.core.subsampling import find_agg_method
from xcube.core.subsampling import get_dataset_subsampling_slices
from xcube.core.subsampling import subsample_dataset


class SubsampleDatasetTest(unittest.TestCase):

    def setUp(self) -> None:
        test_data_1 = np.array([[1, 2, 3, 4, 5, 6],
                                [2, 3, 4, 5, 6, 7],
                                [3, 4, 5, 6, 7, 8],
                                [4, 5, 6, 7, 8, 9],
                                [1, 2, 3, 4, 5, 6]], dtype=np.int16)
        test_data_1 = np.stack([test_data_1, test_data_1 + 10])
        test_data_2 = 0.1 * test_data_1
        self.dataset = new_cube(width=6,  # even
                                height=5,  # odd
                                x_name='x',
                                y_name='y',
                                x_start=0.5,
                                y_start=1.5,
                                x_res=1.,
                                y_res=1.,
                                time_periods=2,
                                crs='CRS84',
                                crs_name='spatial_ref',
                                variables=dict(var_1=test_data_1,
                                               var_2=test_data_2))

    def test_get_dataset_subsampling_slices(self):
        slices = get_dataset_subsampling_slices(self.dataset,
                                                step=2)
        self.assertEqual(
            {
                'var_1': (slice(None, None, None),
                          slice(None, None, 2),
                          slice(None, None, 2)),
                'var_2': (slice(None, None, None),
                          slice(None, None, 2),
                          slice(None, None, 2))
            },
            slices
        )

    def test_subsample_dataset_none(self):
        subsampled_dataset = subsample_dataset(self.dataset,
                                               step=2,
                                               agg_methods=None)
        self.assert_subsampling_ok(
            subsampled_dataset,
            np.array([
                [[1, 3, 5],
                 [3, 5, 7],
                 [1, 3, 5]],

                [[11, 13, 15],
                 [13, 15, 17],
                 [11, 13, 15]]
            ], dtype=np.int16),
            np.array([
                [[0.2, 0.4, 0.6],
                 [0.4, 0.6, 0.8],
                 [0.15, 0.35, 0.55]],

                [[1.2, 1.4, 1.6],
                 [1.4, 1.6, 1.8],
                 [1.15, 1.35, 1.55]]
            ], dtype=np.float64),
            np.array([1., 3., 5.]),
            np.array([2., 4., 6.]),
        )

    def test_subsample_dataset_first(self):
        subsampled_dataset = subsample_dataset(self.dataset,
                                               step=2,
                                               agg_methods='first')
        self.assert_subsampling_ok(
            subsampled_dataset,
            np.array([
                [[1, 3, 5],
                 [3, 5, 7],
                 [1, 3, 5]],

                [[11, 13, 15],
                 [13, 15, 17],
                 [11, 13, 15]]
            ], dtype=np.int16),
            np.array([
                [[0.1, 0.3, 0.5],
                 [0.3, 0.5, 0.7],
                 [0.1, 0.3, 0.5]],

                [[1.1, 1.3, 1.5],
                 [1.3, 1.5, 1.7],
                 [1.1, 1.3, 1.5]]
            ], dtype=np.float64),
            np.array([1., 3., 5.]),
            np.array([2., 4., 6.]),
        )

    def test_subsample_dataset_mean(self):
        subsampled_dataset = subsample_dataset(self.dataset,
                                               step=2,
                                               agg_methods='mean')
        self.assert_subsampling_ok(
            subsampled_dataset,
            np.array([
                [[2, 4, 6],
                 [4, 6, 8],
                 [1, 3, 5]],

                [[12, 14, 16],
                 [14, 16, 18],
                 [11, 13, 15]]
            ], dtype=np.int16),
            np.array([
                [[0.2, 0.4, 0.6],
                 [0.4, 0.6, 0.8],
                 [0.15, 0.35, 0.55]],

                [[1.2, 1.4, 1.6],
                 [1.4, 1.6, 1.8],
                 [1.15, 1.35, 1.55]]
            ], dtype=np.float64),
            np.array([1., 3., 5.]),
            np.array([2., 4., 6.]),
        )

    def test_subsample_dataset_max(self):
        subsampled_dataset = subsample_dataset(self.dataset,
                                               step=2,
                                               agg_methods='max')
        self.assert_subsampling_ok(
            subsampled_dataset,
            np.array([
                [[3, 5, 7],
                 [5, 7, 9],
                 [2, 4, 6]],

                [[13, 15, 17],
                 [15, 17, 19],
                 [12, 14, 16]]
            ], dtype=np.int16),
            np.array([
                [[0.3, 0.5, 0.7],
                 [0.5, 0.7, 0.9],
                 [0.2, 0.4, 0.6]],

                [[1.3, 1.5, 1.7],
                 [1.5, 1.7, 1.9],
                 [1.2, 1.4, 1.6]]
            ], dtype=np.float64),
            np.array([1., 3., 5.]),
            np.array([2., 4., 6.]),
        )

    def assert_subsampling_ok(self,
                              subsampled_dataset: xr.Dataset,
                              expected_var_1: np.ndarray,
                              expected_var_2: np.ndarray,
                              expected_x: np.ndarray,
                              expected_y: np.ndarray):
        self.assertIsInstance(subsampled_dataset, xr.Dataset)
        self.assertIn('spatial_ref',
                      subsampled_dataset)
        self.assertIn('grid_mapping_name',
                      subsampled_dataset.spatial_ref.attrs)

        self.assertIn('x',
                      subsampled_dataset.coords)
        self.assertIn('y',
                      subsampled_dataset.coords)

        np.testing.assert_array_equal(
            subsampled_dataset.x.values,
            expected_x
        )
        np.testing.assert_array_almost_equal(
            subsampled_dataset.y.values,
            expected_y
        )

        self.assertIn('var_1',
                      subsampled_dataset.data_vars)
        self.assertIn('var_2',
                      subsampled_dataset.data_vars)

        self.assertIn('grid_mapping',
                      subsampled_dataset.var_1.attrs)
        self.assertIn('grid_mapping',
                      subsampled_dataset.var_2.attrs)

        self.assertEqual(expected_var_1.dtype,
                         subsampled_dataset.var_1.dtype)
        self.assertEqual(expected_var_2.dtype,
                         subsampled_dataset.var_2.dtype)

        np.testing.assert_array_equal(
            subsampled_dataset.var_1.values,
            expected_var_1
        )
        np.testing.assert_array_almost_equal(
            subsampled_dataset.var_2.values,
            expected_var_2
        )

    # noinspection PyTypeChecker
    def test_find_agg_method(self):
        for m in ('first', 'min', 'max', 'mean', 'median'):
            self.assertEqual(m, find_agg_method(m, 'var_1', np.uint8))
            self.assertEqual(m, find_agg_method(m, 'var_2', np.float32))

        for m in ({'*': 'max'}, {'var_*': 'max'}):
            self.assertEqual('max',
                             find_agg_method(m, 'var_1', np.uint8))
            self.assertEqual('max',
                             find_agg_method(m, 'var_2', np.float32))

        for m in (None, 'auto'):
            self.assertEqual('first', find_agg_method(m, 'var_1', np.uint8))
            self.assertEqual('mean', find_agg_method(m, 'var_2', np.float32))

        for m in ({'*': None}, {'*': 'auto'},
                  {'var_*': None}, {'var_*': 'auto'}):
            self.assertEqual('first', find_agg_method(m, 'var_1', np.uint8))
            self.assertEqual('mean', find_agg_method(m, 'var_2', np.float32))


# test/core/test_tile.py

import unittest

import numpy as np
import pandas as pd
import xarray as xr

from xcube.core.tile import get_var_valid_range
from xcube.core.tile import parse_non_spatial_labels
from xcube.core.tile import _ensure_time_compatible


class GetVarValidRangeTest(unittest.TestCase):
    def test_from_valid_range(self):
        a = xr.DataArray(0, attrs=dict(valid_range=[-1, 1]))
        self.assertEqual((-1, 1),
                         get_var_valid_range(a))

    def test_from_valid_min_max(self):
        a = xr.DataArray(0, attrs=dict(valid_min=-1, valid_max=1))
        self.assertEqual((-1, 1),
                         get_var_valid_range(a))

    def test_from_valid_min(self):
        a = xr.DataArray(0, attrs=dict(valid_min=-1))
        self.assertEqual((-1, np.inf),
                         get_var_valid_range(a))

    def test_from_valid_max(self):
        a = xr.DataArray(0, attrs=dict(valid_max=1))
        self.assertEqual((-np.inf, 1),
                         get_var_valid_range(a))

    def test_from_nothing(self):
        a = xr.DataArray(0)
        self.assertEqual(None,
                         get_var_valid_range(a))


class ParseNonSpatialLabelsTest(unittest.TestCase):
    time = xr.DataArray(np.array(['2021-06-07 10:20:45',
                                  '2021-06-08 13:28:16',
                                  '2021-06-09 11:10:21'],
                                 dtype='datetime64'),
                        dims='time')
    lat = xr.DataArray(np.array([60, 61, 62], dtype=np.float64),
                       dims='lat')
    lon = xr.DataArray(np.array([4, 5, 6], dtype=np.float64),
                       dims='lon')
    coords = dict(time=time, lat=lat, lon=lon)

    dims = ('time', 'lat', 'lon')

    def test_use_first_not_given(self):
        labels = parse_non_spatial_labels(dict(),
                                          dims=self.dims,
                                          coords=self.coords)
        self.assertEqual(
            dict(time=np.array('2021-06-07 10:20:45', dtype='datetime64')),
            labels
        )

    def test_use_last_if_current(self):
        labels = parse_non_spatial_labels(dict(time='current'),
                                          dims=self.dims,
                                          coords=self.coords)
        self.assertEqual(
            dict(time=np.array('2021-06-09 11:10:21', dtype='datetime64')),
            labels
        )

    def test_use_given_if_value_given(self):
        labels = parse_non_spatial_labels(dict(time='2021-06-08'),
                                          dims=self.dims,
                                          coords=self.coords)
        self.assertEqual(
            dict(time=np.array('2021-06-08', dtype='datetime64')),
            labels
        )

    def test_use_average_if_range_given(self):
        labels = parse_non_spatial_labels(dict(time='2021-06-06/2021-06-08'),
                                          dims=self.dims,
                                          coords=self.coords)
        self.assertEqual(
            dict(time=np.array('2021-06-07', dtype='datetime64')),
            labels
        )

    def test_use_slice_if_range_given(self):
        labels = parse_non_spatial_labels(dict(time='2021-06-06/2021-06-08'),
                                          dims=self.dims,
                                          coords=self.coords,
                                          allow_slices=True)
        self.assertEqual(
            dict(time=slice(np.array('2021-06-06', dtype='datetime64'),
                            np.array('2021-06-08', dtype='datetime64'))),
            labels
        )

    def test_invalid(self):
        with self.assertRaises(ValueError) as cm:
            parse_non_spatial_labels(dict(time='jetzt'),
                                     dims=self.dims,
                                     coords=self.coords)
        self.assertEqual("'jetzt' is not a valid value for dimension 'time'",
                         f'{cm.exception}')

    def test_ensure_timezone_naive(self):
        da_tznaive = xr.DataArray(
            np.zeros((3,3,3)),
            coords=self.coords,
            dims=self.dims)
        labels = parse_non_spatial_labels(dict(time='2000-01-02T00:00:00Z'),
                                          dims=da_tznaive.dims,
                                          coords=da_tznaive.coords,
                                          var=da_tznaive)
        self.assertIsNone(pd.Timestamp(labels['time']).tzinfo)


class EnsureTimeCompatibleTest(unittest.TestCase):
    da_tznaive = xr.DataArray(
        np.arange(1, 4),
        coords=dict(time=np.arange('2000-01-01', '2000-01-04',
                                   dtype=np.datetime64)),
        dims=['time'])
    da_tzaware = xr.DataArray(
        np.arange(1, 4),
        coords=dict(time=pd.date_range("2000-01-01", "2000-01-03", tz='UTC')),
        dims=['time'])
    labels_tznaive = dict(time='2000-01-02')
    labels_tzaware = dict(time='2000-01-02T00:00:00Z')

    def test_both_tznaive(self):
        self.assertEqual(self.labels_tznaive,
                         _ensure_time_compatible(self.da_tznaive,
                                                 self.labels_tznaive))

    def test_both_tzaware(self):
        self.assertEqual(self.labels_tzaware,
                         _ensure_time_compatible(self.da_tzaware,
                                                 self.labels_tzaware))

    def test_tznaive_array_tzaware_indexer(self):
        self.assertTrue(
            _are_times_equal(
                self.labels_tznaive,
                _ensure_time_compatible(self.da_tznaive,
                                        self.labels_tzaware)))


def _are_times_equal(labels1, labels2):
    return pd.Timestamp(labels1['time']) == pd.Timestamp(labels2['time'])


# test/core/test_tilingscheme.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.


import unittest

from xcube.core.tilingscheme import EARTH_CIRCUMFERENCE_WGS84
from xcube.core.tilingscheme import TilingScheme
from xcube.core.tilingscheme import get_num_levels
from xcube.core.tilingscheme import get_unit_factor
from xcube.core.tilingscheme import subdivide_size


class TilingSchemeTest(unittest.TestCase):

    def test_for_crs_geographic(self):
        self.assertIs(TilingScheme.GEOGRAPHIC,
                      TilingScheme.for_crs('EPSG:4326'))
        self.assertIs(TilingScheme.GEOGRAPHIC,
                      TilingScheme.for_crs(
                          'urn:ogc:def:crs:OGC:1.3:EPSG::4326'
                      ))
        self.assertIs(TilingScheme.GEOGRAPHIC,
                      TilingScheme.for_crs('CRS84'))
        self.assertIs(TilingScheme.GEOGRAPHIC,
                      TilingScheme.for_crs('urn:ogc:def:crs:OGC:1.3:CRS84'))
        self.assertIs(TilingScheme.GEOGRAPHIC,
                      TilingScheme.for_crs('WGS84'))

        self.assertIs(TilingScheme.GEOGRAPHIC,
                      TilingScheme.for_crs(TilingScheme.GEOGRAPHIC.crs))

        tiling_scheme = TilingScheme.for_crs('CRS84', tile_size=512)
        self.assertIsNot(TilingScheme.GEOGRAPHIC, tiling_scheme)
        self.assertEqual(512, tiling_scheme.tile_size)

    def test_for_crs_web_mercator(self):
        self.assertIs(TilingScheme.WEB_MERCATOR,
                      TilingScheme.for_crs('EPSG:3857'))
        self.assertIs(TilingScheme.WEB_MERCATOR,
                      TilingScheme.for_crs(
                          'urn:ogc:def:crs:OGC:1.3:EPSG::3857')
                      )

        self.assertIs(TilingScheme.WEB_MERCATOR,
                      TilingScheme.for_crs(TilingScheme.WEB_MERCATOR.crs))

        tiling_scheme = TilingScheme.for_crs('EPSG:3857', tile_size=512)
        self.assertIsNot(TilingScheme.WEB_MERCATOR, tiling_scheme)
        self.assertEqual(512, tiling_scheme.tile_size)

    def test_properties_geographic(self):
        tiling_scheme = TilingScheme.GEOGRAPHIC

        self.assertEqual((2, 1), tiling_scheme.num_level_zero_tiles)
        self.assertEqual('CRS84', tiling_scheme.crs_name)
        self.assertEqual(256, tiling_scheme.tile_size)
        self.assertEqual(360, tiling_scheme.map_width)
        self.assertEqual(180, tiling_scheme.map_height)
        self.assertEqual(None, tiling_scheme.min_level)
        self.assertEqual(None, tiling_scheme.max_level)
        self.assertEqual(None, tiling_scheme.num_levels)
        self.assertEqual((-180, -90, 180, 90), tiling_scheme.map_extent)
        self.assertEqual((-180, 90), tiling_scheme.map_origin)

    def test_properties_web_mercator(self):
        tiling_scheme = TilingScheme.WEB_MERCATOR

        self.assertEqual((1, 1), tiling_scheme.num_level_zero_tiles)
        self.assertEqual('EPSG:3857', tiling_scheme.crs_name)
        self.assertEqual(256, tiling_scheme.tile_size)
        self.assertEqual(EARTH_CIRCUMFERENCE_WGS84, tiling_scheme.map_width)
        self.assertEqual(EARTH_CIRCUMFERENCE_WGS84, tiling_scheme.map_height)
        self.assertEqual(None, tiling_scheme.min_level)
        self.assertEqual(None, tiling_scheme.max_level)
        self.assertEqual(None, tiling_scheme.num_levels)
        half = EARTH_CIRCUMFERENCE_WGS84 / 2
        self.assertEqual((-half, -half, half, half), tiling_scheme.map_extent)
        self.assertEqual((-half, half), tiling_scheme.map_origin)

    def test_tile_extent_web_mercator(self):
        tiling_scheme = TilingScheme.WEB_MERCATOR

        half = EARTH_CIRCUMFERENCE_WGS84 / 2

        self.assertEqual((-half, -half, half, half),
                         tiling_scheme.get_tile_extent(0, 0, 0))

        self.assertEqual((-half, 0.0, 0.0, half),
                         tiling_scheme.get_tile_extent(0, 0, 1))
        self.assertEqual((0.0, 0.0, half, half),
                         tiling_scheme.get_tile_extent(1, 0, 1))
        self.assertEqual((-half, -half, 0.0, 0.0),
                         tiling_scheme.get_tile_extent(0, 1, 1))
        self.assertEqual((0.0, -half, half, 0.0),
                         tiling_scheme.get_tile_extent(1, 1, 1))

    def test_derive_geographic(self):
        tiling_scheme = TilingScheme.GEOGRAPHIC
        derived_tile_grid = tiling_scheme.derive(min_level=3, max_level=9)
        self.assertIsInstance(derived_tile_grid, TilingScheme)
        self.assertIsNot(tiling_scheme, derived_tile_grid)
        self.assertEqual(3, derived_tile_grid.min_level)
        self.assertEqual(9, derived_tile_grid.max_level)
        self.assertEqual(10, derived_tile_grid.num_levels)

    def test_get_resolutions_geographic(self):
        tiling_scheme = TilingScheme.GEOGRAPHIC

        with self.assertRaises(ValueError) as cm:
            tiling_scheme.get_resolutions()
        self.assertEqual('max_value must be given', f'{cm.exception}')

        resolutions = tiling_scheme.get_resolutions(max_level=6)
        self.assertEqual([0.703125,
                          0.3515625,
                          0.17578125,
                          0.087890625,
                          0.0439453125,
                          0.02197265625,
                          0.010986328125],
                         resolutions)

        resolutions = tiling_scheme.get_resolutions(min_level=4, max_level=6)
        self.assertEqual([0.0439453125,
                          0.02197265625,
                          0.010986328125],
                         resolutions)

        resolutions = tiling_scheme.derive(max_level=6). \
            get_resolutions()
        self.assertEqual([0.703125,
                          0.3515625,
                          0.17578125,
                          0.087890625,
                          0.0439453125,
                          0.02197265625,
                          0.010986328125],
                         resolutions)

        resolutions = tiling_scheme.derive(min_level=4, max_level=6). \
            get_resolutions()
        self.assertEqual([0.0439453125,
                          0.02197265625,
                          0.010986328125],
                         resolutions)

    def test_tile_extent_geographic(self):
        tiling_scheme = TilingScheme.GEOGRAPHIC

        self.assertEqual((-180, -90, 0, 90),
                         tiling_scheme.get_tile_extent(0, 0, 0))
        self.assertEqual((0, -90, 180, 90),
                         tiling_scheme.get_tile_extent(1, 0, 0))

        self.assertEqual((-180, 0, -90, 90),
                         tiling_scheme.get_tile_extent(0, 0, 1))
        self.assertEqual((-90, 0, 0, 90),
                         tiling_scheme.get_tile_extent(1, 0, 1))
        self.assertEqual((0, 0, 90, 90),
                         tiling_scheme.get_tile_extent(2, 0, 1))
        self.assertEqual((90, 0, 180, 90),
                         tiling_scheme.get_tile_extent(3, 0, 1))
        self.assertEqual((-180, -90, -90, 0),
                         tiling_scheme.get_tile_extent(0, 1, 1))
        self.assertEqual((-90, -90, 0, 0),
                         tiling_scheme.get_tile_extent(1, 1, 1))
        self.assertEqual((0, -90, 90, 0),
                         tiling_scheme.get_tile_extent(2, 1, 1))
        self.assertEqual((90, -90, 180, 0),
                         tiling_scheme.get_tile_extent(3, 1, 1))

    def test_get_resolutions_level_web_mercator(self):
        tiling_scheme = TilingScheme.WEB_MERCATOR

        # # This useful for debugging failing tests
        # resolutions = tiling_scheme.get_resolutions(unit_name='meter',
        #                                             max_level=20)
        # for level, res in enumerate(resolutions):
        #     print(level, res)

        args = [10, 20, 40, 80, 160], 'meter'

        get_level = tiling_scheme.get_resolutions_level
        self.assertEqual(4, get_level(0, *args))
        self.assertEqual(4, get_level(1, *args))
        self.assertEqual(4, get_level(2, *args))
        self.assertEqual(4, get_level(3, *args))
        self.assertEqual(4, get_level(4, *args))
        self.assertEqual(4, get_level(5, *args))
        self.assertEqual(4, get_level(6, *args))
        self.assertEqual(4, get_level(7, *args))
        self.assertEqual(4, get_level(8, *args))
        self.assertEqual(4, get_level(9, *args))
        self.assertEqual(4, get_level(10, *args))
        self.assertEqual(3, get_level(11, *args))
        self.assertEqual(2, get_level(12, *args))
        self.assertEqual(1, get_level(13, *args))
        self.assertEqual(0, get_level(14, *args))
        self.assertEqual(0, get_level(15, *args))
        self.assertEqual(0, get_level(16, *args))
        self.assertEqual(0, get_level(17, *args))
        self.assertEqual(0, get_level(18, *args))
        self.assertEqual(0, get_level(19, *args))

    def test_get_levels_for_resolutions_web_mercator(self):
        tiling_scheme = TilingScheme.WEB_MERCATOR

        self.assertEqual(
            (10, 14),
            tiling_scheme.get_levels_for_resolutions(
                [10, 20, 40, 80, 160],
                'meter'
            )
        )

    def test_get_resolutions_level_geographic(self):
        tiling_scheme = TilingScheme.GEOGRAPHIC

        # # This useful for debugging failing tests
        # resolutions = tiling_scheme.get_resolutions(unit_name='degree',
        #                                             max_level=6)
        # for level, res in enumerate(resolutions):
        #     print(level, res)

        num_ds_levels = 6
        ds_resolutions = [
            180 / 256 / 2 ** i for i in reversed(range(num_ds_levels))
        ]

        args = ds_resolutions, 'degree'

        get_level = tiling_scheme.get_resolutions_level
        self.assertEqual(5, get_level(0, *args))
        self.assertEqual(4, get_level(1, *args))
        self.assertEqual(3, get_level(2, *args))
        self.assertEqual(2, get_level(3, *args))
        self.assertEqual(1, get_level(4, *args))
        self.assertEqual(0, get_level(5, *args))
        self.assertEqual(0, get_level(6, *args))
        self.assertEqual(0, get_level(7, *args))

    def test_get_levels_for_resolutions_geographic(self):
        tiling_scheme = TilingScheme.GEOGRAPHIC

        num_ds_levels = 10
        ds_resolutions = [
            180 / 256 / 2 ** i for i in reversed(range(num_ds_levels))
        ]

        # Test up to num_ds_levels
        self.assertEqual(
            (0, num_ds_levels - 1),
            tiling_scheme.get_levels_for_resolutions(
                ds_resolutions,
                'degrees'
            )
        )

        # Test single level 5
        self.assertEqual(
            (5, 5),
            tiling_scheme.get_levels_for_resolutions(
                [180 / 256 / 2 ** 5],
                'degrees'
            )
        )

        # Test empty
        with self.assertRaises(ValueError):
            tiling_scheme.get_levels_for_resolutions(
                [],
                'degrees'
            )


class TilingSchemeHelpersTest(unittest.TestCase):

    def test_subdivide_size(self):
        self.assertEqual(
            [(2048, 1024), (1024, 512), (512, 256)],
            subdivide_size((2048, 1024), (256, 256))
        )
        self.assertEqual(
            [(360, 180)],
            subdivide_size((360, 180), (256, 256))
        )
        self.assertEqual(
            [(7200, 3600), (3600, 1800), (1800, 900), (900, 450), (450, 225)],
            subdivide_size((7200, 3600), (256, 256))
        )

    def test_get_num_levels(self):
        self.assertEqual(3, get_num_levels((2048, 1024), (256, 256)))
        self.assertEqual(1, get_num_levels((360, 180), (256, 256)))
        self.assertEqual(5, get_num_levels((7200, 3600), (256, 256)))

    def test_get_unit_factor(self):
        self.assertAlmostEqual(1, get_unit_factor('m', 'm'))
        self.assertAlmostEqual(1, get_unit_factor('meters', 'm'))
        self.assertAlmostEqual(1, get_unit_factor('deg', 'degree'))
        self.assertAlmostEqual(111319.49079327358,
                               get_unit_factor('deg', 'm'))
        self.assertAlmostEqual(8.983152841195214e-06,
                               get_unit_factor('m', 'degree'))
        with self.assertRaises(ValueError):
            get_unit_factor('cm', 'm')
        with self.assertRaises(ValueError):
            get_unit_factor('m', 'mdeg')


# test/core/test_timecoord.py

import unittest

import numpy as np
import pandas as pd

from test.sampledata import create_highroc_dataset
from xcube.core.new import new_cube
from xcube.core.timecoord import add_time_coords
from xcube.core.timecoord import from_time_in_days_since_1970
from xcube.core.timecoord import get_end_time_from_attrs
from xcube.core.timecoord import get_start_time_from_attrs
from xcube.core.timecoord import get_time_range_from_attrs
from xcube.core.timecoord import get_time_range_from_data
from xcube.core.timecoord import timestamp_to_iso_string
from xcube.core.timecoord import to_time_in_days_since_1970
# from xcube.core.timecoord import find_datetime_format
# from xcube.core.timecoord import get_timestamp_from_string
# from xcube.core.timecoord import get_timestamps_from_string


class AddTimeCoordsTest(unittest.TestCase):

    def test_add_time_coords_point(self):
        dataset = create_highroc_dataset()
        dataset_with_time = add_time_coords(dataset, (365 * 47 + 20, 365 * 47 + 20))
        self.assertIsNot(dataset_with_time, dataset)
        self.assertIn('time', dataset_with_time)
        self.assertEqual(dataset_with_time.time.shape, (1,))
        self.assertNotIn('time_bnds', dataset_with_time)

    def test_add_time_coords_range(self):
        dataset = create_highroc_dataset()
        dataset_with_time = add_time_coords(dataset, (365 * 47 + 20, 365 * 47 + 21))
        self.assertIsNot(dataset_with_time, dataset)
        self.assertIn('time', dataset_with_time)
        self.assertEqual(dataset_with_time.time.shape, (1,))
        self.assertIn('time_bnds', dataset_with_time)
        self.assertEqual(dataset_with_time.time_bnds.shape, (1, 2))

    def test_to_time_in_days_since_1970(self):
        self.assertEqual(17324.5,
                         to_time_in_days_since_1970('201706071200'))
        self.assertEqual(17325.5,
                         to_time_in_days_since_1970('201706081200'))
        self.assertEqual(17690.5,
                         to_time_in_days_since_1970('2018-06-08 12:00'))
        self.assertEqual(17690.5,
                         to_time_in_days_since_1970('2018-06-08T12:00'))
        self.assertEqual(18173.42625622898,
                         to_time_in_days_since_1970('04-OCT-2019 10:13:48.538184'))

    def test_from_time_in_days_since_1970(self):
        self.assertEqual('2017-06-07T12:00:00.000000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('201706071200'))))
        self.assertEqual('2017-06-08T12:00:00.000000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('201706081200'))))
        self.assertEqual('2018-06-08T12:00:00.000000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('2018-06-08 12:00'))))
        self.assertEqual('2018-06-08T12:00:00.000000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('2018-06-08T12:00'))))
        self.assertEqual('2019-10-04T10:13:48.538000000',
                         str(from_time_in_days_since_1970(
                             to_time_in_days_since_1970('04-OCT-2019 10:13:48.538184'))))


class GetTimeRangeTest(unittest.TestCase):

    def test_get_time_range_from_data(self):
        cube = new_cube(drop_bounds=True)
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_with_irregular_data(self):
        cube = new_cube(drop_bounds=True,
                        time_freq='M')
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_with_irregular_data_and_no_metadata(self):
        cube = new_cube(drop_bounds=True,
                        time_freq='M')
        cube.attrs.pop('time_coverage_start')
        cube.attrs.pop('time_coverage_end')
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_cftime(self):
        cube = new_cube(drop_bounds=True,
                        use_cftime=True,
                        time_dtype=None)
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_with_irregular_cftime_data(self):
        cube = new_cube(drop_bounds=True,
                        time_freq='M',
                        use_cftime=True,
                        time_dtype=None)
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_with_irregular_cftime_data_and_no_metadata(self):
        cube = new_cube(drop_bounds=True,
                        time_freq='M',
                        use_cftime=True,
                        time_dtype=None)
        cube.attrs.pop('time_coverage_start')
        cube.attrs.pop('time_coverage_end')
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_time_named_t(self):
        cube = new_cube(drop_bounds=True, time_name='t')
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_additional_t_variable(self):
        import xarray as xr
        start_time_data = pd.date_range(start='2010-01-03T12:00:00',
                                        periods=5,
                                        freq='5D').values.astype(dtype='datetime64[s]')
        start_time = xr.DataArray(start_time_data, dims='time')
        end_time_data = pd.date_range(start='2010-01-07T12:00:00',
                                      periods=5,
                                      freq='5D').values.astype(dtype='datetime64[s]')
        end_time = xr.DataArray(end_time_data, dims='time')
        cube = new_cube(drop_bounds=True,
                        time_start='2010-01-05T12:00:00',
                        time_freq='5D',
                        variables=dict(start_time=start_time, end_time=end_time))
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-03T12:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-27T12:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_start_and_end_time_arrays(self):
        cube = new_cube(drop_bounds=True,
                        use_cftime=True,
                        time_dtype=None)
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_data_bounds(self):
        cube = new_cube()
        time_range = get_time_range_from_data(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_time_range_from_attrs(self):
        cube = new_cube()
        time_range = get_time_range_from_attrs(cube)
        self.assertIsNotNone(time_range)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())

    def test_get_start_time_from_attrs(self):
        cube = new_cube()
        start_time = get_start_time_from_attrs(cube)
        self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(start_time).isoformat())

    def test_get_end_time_from_attrs(self):
        cube = new_cube()
        end_time = get_end_time_from_attrs(cube)
        self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(end_time).isoformat())


class TimestampToIsoStringTest(unittest.TestCase):
    def test_it_with_default_res(self):
        self.assertEqual("2018-09-05T00:00:00Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05")))
        self.assertEqual("2018-09-05T10:35:42Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42")))
        self.assertEqual("2018-09-05T10:35:42Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164")))
        self.assertEqual("2019-10-04T10:13:49Z",
                         timestamp_to_iso_string(pd.to_datetime("04-OCT-2019 10:13:48.538184")))

    def test_it_with_h_res(self):
        self.assertEqual("2018-09-05T00:00:00Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05"),
                                                 freq="H"))
        self.assertEqual("2018-09-05T11:00:00Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42"),
                                                 freq="H"))
        self.assertEqual("2018-09-05T11:00:00Z",
                         timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164"),
                                                 freq="H"))
        self.assertEqual("2019-10-04T10:00:00Z",
                         timestamp_to_iso_string(pd.to_datetime("04-OCT-2019 10:13:48.538184"),
                                                 freq="H"))


# class TimeStampsTest(unittest.TestCase):
#
#     def test_find_datetime_format(self):
#         dt_format, start_index, end_index = find_datetime_format('ftze20140305131415dgs')
#         self.assertEqual('%Y%m%d%H%M%S', dt_format)
#         self.assertEqual(4, start_index)
#         self.assertEqual(18, end_index)
#
#         dt_format, start_index, end_index = find_datetime_format('ftze201403051314dgs')
#         self.assertEqual('%Y%m%d%H%M', dt_format)
#         self.assertEqual(4, start_index)
#         self.assertEqual(16, end_index)
#
#         dt_format, start_index, end_index = find_datetime_format('ft2ze20140307dgs')
#         self.assertEqual('%Y%m%d', dt_format)
#         self.assertEqual(5, start_index)
#         self.assertEqual(13, end_index)
#
#         dt_format, start_index, end_index = find_datetime_format('ft2ze201512dgs')
#         self.assertEqual('%Y%m', dt_format)
#         self.assertEqual(5, start_index)
#         self.assertEqual(11, end_index)
#
#         dt_format, start_index, end_index = find_datetime_format('ft2s6ze2016dgs')
#         self.assertEqual('%Y', dt_format)
#         self.assertEqual(7, start_index)
#         self.assertEqual(11, end_index)
#
#     def test_get_timestamp_from_string(self):
#         timestamp = get_timestamp_from_string('ftze20140305131415dg0023s')
#         self.assertEqual(pd.Timestamp('2014-03-05T13:14:15'), timestamp)
#
#         timestamp = get_timestamp_from_string('ftze201403051314dgs')
#         self.assertEqual(pd.Timestamp('2014-03-05T13:14:00'), timestamp)
#
#         timestamp = get_timestamp_from_string('ftze20140305dgs')
#         self.assertEqual(pd.Timestamp('2014-03-05T00:00:00'), timestamp)
#
#         timestamp = get_timestamp_from_string('ftze201403dgs')
#         self.assertEqual(pd.Timestamp('2014-03-01T00:00:00'), timestamp)
#
#         timestamp = get_timestamp_from_string('ftze2014dgs')
#         self.assertEqual(pd.Timestamp('2014-01-01T00:00:00'), timestamp)
#
#     def test_get_timestamps_from_string(self):
#         timestamp_1, timestamp_2 = get_timestamps_from_string(
#             '20020401-20020406-ESACCI-L3C_AEROSOL-AEX-GOMOS_ENVISAT-AERGOM_5days-fv2.19.nc')
#         self.assertEqual(pd.Timestamp('2002-04-01'), timestamp_1)
#         self.assertEqual(pd.Timestamp('2002-04-06'), timestamp_2)
#
#         timestamp_1, timestamp_2 = get_timestamps_from_string(
#             '20020401-ESACCI-L3C_AEROSOL-AEX-GOMOS_ENVISAT-AERGOM_5days-fv2.19.nc')
#         self.assertEqual(pd.Timestamp('2002-04-01'), timestamp_1)
#         self.assertIsNone(timestamp_2)


# test/core/test_timeseries.py

import unittest
from typing import Set

import numpy as np
import pyproj
import xarray as xr

from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.timeseries import get_time_series

POINT_GEOMETRY = dict(type='Point', coordinates=[20.0, 10.0])
POLYGON_GEOMETRY = dict(type='Polygon', coordinates=[[[20.0, 10.0],
                                                      [20.0, 20.0],
                                                      [10.0, 20.0],
                                                      [10.0, 10.0],
                                                      [20.0, 10.0]]])


class GetTimeSeriesTest(unittest.TestCase):

    def test_point(self):
        ts_ds = get_time_series(self.cube, geometry=POINT_GEOMETRY)
        self.assert_dataset_ok(ts_ds, 1, {'A', 'B'})

    def test_polygon(self):
        ts_ds = get_time_series(self.cube, geometry=POLYGON_GEOMETRY)
        self.assert_dataset_ok(ts_ds, 100, {'A_mean', 'B_mean'})

    def test_polygon_with_grid_mapping(self):
        crs1 = pyproj.CRS.from_string('EPSG:4326')  # = Geographic
        crs2 = pyproj.CRS.from_string('+proj=mill +lat_0=0 +lon_0=0 +x_0=0 +y_0=0 +R_A +datum=WGS84 +units=m +no_defs')  # = World Miller
        t = pyproj.Transformer.from_crs(crs1, crs2, always_xy=True)
        cube = self.cube.rename(dict(lat='y', lon='x'))
        x, _ = t.transform(
            cube.x,
            np.zeros(cube.x.size)
        )
        _, y = t.transform(
            np.zeros(cube.y.size),
            cube.y,
        )
        cube = cube.assign_coords(
            x=xr.DataArray(x, dims='x'),
            y=xr.DataArray(y, dims='y'),
        )
        cube = cube.assign(crs=xr.DataArray(0, attrs=crs2.to_cf()))
        grid_mapping = GridMapping.from_dataset(cube)
        ts_ds = get_time_series(cube,
                                grid_mapping=grid_mapping,
                                geometry=POLYGON_GEOMETRY)
        self.assertIsInstance(ts_ds, xr.Dataset)
        # TODO (forman): we must actually get a valid
        #  result here, but we don't
        # self.assert_dataset_ok(ts_ds, 100, {'A_mean', 'B_mean'})

    def test_polygon_deprecated_cube_asserted(self):
        ts_ds = get_time_series(self.cube, geometry=POLYGON_GEOMETRY,
                                cube_asserted=True)
        self.assert_dataset_ok(ts_ds, 100, {'A_mean', 'B_mean'})

    def test_polygon_agg_median_mean_std(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                agg_methods=['mean', 'std', 'median'])
        self.assert_dataset_ok(
            ts_ds, 100,
            {'A_mean', 'A_median', 'A_std', 'B_mean', 'B_median', 'B_std'}
        )

    def test_polygon_agg_median_mean_std_groupby(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                agg_methods=['mean', 'std', 'median'],
                                use_groupby=True)
        self.assert_dataset_ok(
            ts_ds, 100,
            {'A_mean', 'A_median', 'A_std', 'B_mean', 'B_median', 'B_std'}
        )

    def test_polygon_agg_mean_count(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                agg_methods=['mean', 'count'])
        self.assert_dataset_ok(
            ts_ds, 100,
            {'A_mean', 'A_count', 'B_mean', 'B_count'}
        )

    def test_polygon_agg_mean_std_var_subs(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                var_names=['B'],
                                agg_methods=['mean', 'std'])
        self.assert_dataset_ok(ts_ds, 100, {'B_mean', 'B_std'})

    def test_no_vars(self):
        ts_ds = get_time_series(self.cube,
                                geometry=POLYGON_GEOMETRY,
                                var_names=[])
        self.assertIsNone(ts_ds)

    def test_illegal_agg_methods(self):
        with self.assertRaises(ValueError) as cm:
            get_time_series(self.cube,
                            geometry=POLYGON_GEOMETRY,
                            agg_methods=['mean', 'median', 'stdev'])
        self.assertEqual('invalid aggregation method: stdev',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            get_time_series(self.cube,
                            geometry=POLYGON_GEOMETRY,
                            agg_methods=['median', 'stdev', 'avg'])
        self.assertEqual('invalid aggregation methods: avg, stdev',
                         f'{cm.exception}')

    def setUp(self):
        shape = 25, 180, 360
        dims = 'time', 'lat', 'lon'
        self.ts_a = np.linspace(1, 25, 25)
        self.ts_a_mean = np.linspace(1, 25, 25)
        self.ts_a_count = np.array(25 * [100])
        self.ts_a_std = np.array(25 * [0.0])
        self.ts_b = np.linspace(0, 1, 25)
        self.ts_b_mean = np.linspace(0, 1, 25)
        self.ts_b_count = np.array(25 * [100])
        self.ts_b_std = np.array(25 * [0.0])
        self.cube = new_cube(
            time_periods=25,
            variables=dict(
                A=xr.DataArray(
                    np.broadcast_to(self.ts_a.reshape((25, 1, 1)), shape),
                    dims=dims),
                B=xr.DataArray(
                    np.broadcast_to(self.ts_b.reshape((25, 1, 1)), shape),
                    dims=dims)
            )
        )
        self.cube = self.cube.chunk(chunks=dict(time=1, lat=180, lon=180))

    def assert_dataset_ok(self, ts_ds: xr.Dataset,
                          expected_max_number_of_observations: int = 0,
                          expected_var_names: Set = None):
        expected_var_names = expected_var_names or set()
        self.assertIsNotNone(ts_ds)
        self.assertEqual(expected_max_number_of_observations, ts_ds.attrs.get('max_number_of_observations'))
        self.assert_variable_ok(ts_ds, 'A', expected_var_names, self.ts_a)
        self.assert_variable_ok(ts_ds, 'A_mean', expected_var_names, self.ts_a_mean)
        self.assert_variable_ok(ts_ds, 'A_count', expected_var_names, self.ts_a_count)
        self.assert_variable_ok(ts_ds, 'A_std', expected_var_names, self.ts_a_std)
        self.assert_variable_ok(ts_ds, 'B', expected_var_names, self.ts_b)
        self.assert_variable_ok(ts_ds, 'B_mean', expected_var_names, self.ts_b_mean)
        self.assert_variable_ok(ts_ds, 'B_count', expected_var_names, self.ts_b_count)
        self.assert_variable_ok(ts_ds, 'B_std', expected_var_names, self.ts_b_std)

    def assert_variable_ok(self, ts_ds, name, expected_var_names: Set, expected_values):
        if name in expected_var_names:
            self.assertIn(name, ts_ds)
            self.assertEqual(('time',), ts_ds[name].dims)
            self.assertEqual(25, ts_ds[name].size)
            np.testing.assert_almost_equal(ts_ds[name].values, expected_values)
        else:
            self.assertNotIn(name, ts_ds)


# test/core/test_timeslice.py

import unittest

import numpy as np
import xarray as xr
import zarr

from test.core.diagnosticstore import DiagnosticStore, logging_observer
from xcube.core.chunk import chunk_dataset
from xcube.core.dsio import rimraf
from xcube.core.new import new_cube
from xcube.core.timeslice import find_time_slice, append_time_slice, insert_time_slice, replace_time_slice


class TimeSliceTest(unittest.TestCase):
    CUBE_PATH = 'test-cube.zarr'
    CUBE_PATH_2 = 'test-cube-2.zarr'

    def setUp(self) -> None:
        rimraf(self.CUBE_PATH)
        rimraf(self.CUBE_PATH_2)

    def tearDown(self) -> None:
        rimraf(self.CUBE_PATH)
        rimraf(self.CUBE_PATH_2)

    def write_slice(self, start_date):
        self.write_cube(start_date, 1)

    def make_slice(self, start_date) -> xr.Dataset:
        return self.make_cube(start_date, 1)

    def write_cube(self, start_date, num_days: int):
        cube = self.make_cube(start_date, num_days)
        cube.to_zarr(self.CUBE_PATH)

    def make_cube(self, start_date, num_days: int) -> xr.Dataset:
        cube = new_cube(time_periods=num_days,
                        time_freq='1D',
                        time_start=start_date,
                        variables=dict(precipitation=0.1,
                                       temperature=270.5,
                                       soil_moisture=0.2))
        chunk_sizes = dict(time=1, lat=90, lon=90)
        cube = chunk_dataset(cube, chunk_sizes, format_name='zarr')
        return cube

    def test_find_time_slice(self):
        self.write_cube('2019-01-01', 10)

        # Cube does not exists --> write new
        result = find_time_slice(self.CUBE_PATH_2, np.datetime64('2018-12-30T13:00'))
        self.assertEqual((-1, 'create'), result)

        # Before first step --> insert before 0
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2018-12-30T13:00'))
        self.assertEqual((0, 'insert'), result)

        # After first step --> insert before 1
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2019-01-02'))
        self.assertEqual((1, 'insert'), result)

        # In-between --> insert before 5
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2019-01-06T10:00:00'))
        self.assertEqual((5, 'insert'), result)

        # In-between at existing time-stamp --> replace 5
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2019-01-06T12:00:00'))
        self.assertEqual((5, 'replace'), result)

        # After last step --> append
        result = find_time_slice(self.CUBE_PATH, np.datetime64('2019-01-12'))
        self.assertEqual((-1, 'append'), result)

    def test_append_time_slice(self):
        self.write_slice('2019-01-01T14:30')

        append_time_slice(self.CUBE_PATH, self.make_slice('2019-01-02T01:14'))
        append_time_slice(self.CUBE_PATH, self.make_slice('2019-01-02T05:16'))
        append_time_slice(self.CUBE_PATH, self.make_slice('2019-01-03T03:50'))

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(['2019-01-02T02:30',
                             '2019-01-02T13:14',
                             '2019-01-02T17:16',
                             '2019-01-03T15:50'], dtype=cube.time.dtype)
        self.assertEqual(4, cube.time.size)
        self.assertEqual(None, cube.time.chunks)
        np.testing.assert_equal(cube.time.values, expected)

    def test_insert_time_slice(self):
        self.write_cube('2019-01-02T00:00:00', 10)

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(
            ['2019-01-02T12:00:00', '2019-01-03T12:00:00',
             '2019-01-04T12:00:00', '2019-01-05T12:00:00',
             '2019-01-06T12:00:00', '2019-01-07T12:00:00',
             '2019-01-08T12:00:00', '2019-01-09T12:00:00',
             '2019-01-10T12:00:00', '2019-01-11T12:00:00'],
            dtype=cube.time.dtype)
        np.testing.assert_equal(cube.time.values, expected)

        insert_time_slice(self.CUBE_PATH, 5,
                          self.make_slice('2019-01-06T02:00:00'))

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(
            ['2019-01-02T12:00:00', '2019-01-03T12:00:00',
             '2019-01-04T12:00:00', '2019-01-05T12:00:00',
             '2019-01-06T12:00:00', '2019-01-06T14:00:00', '2019-01-07T12:00:00',
             '2019-01-08T12:00:00', '2019-01-09T12:00:00',
             '2019-01-10T12:00:00', '2019-01-11T12:00:00'],
            dtype=cube.time.dtype)
        np.testing.assert_equal(cube.time.values, expected)

        insert_time_slice(self.CUBE_PATH, 10, self.make_slice('2019-01-10T02:00'))
        insert_time_slice(self.CUBE_PATH, 0, self.make_slice('2019-01-01T02:00'))

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(['2019-01-01T14:00', '2019-01-02T12:00',
                             '2019-01-03T12:00', '2019-01-04T12:00',
                             '2019-01-05T12:00', '2019-01-06T12:00',
                             '2019-01-06T14:00', '2019-01-07T12:00',
                             '2019-01-08T12:00', '2019-01-09T12:00',
                             '2019-01-10T12:00', '2019-01-10T14:00',
                             '2019-01-11T12:00'], dtype=cube.time.dtype)
        actual = cube.time.values
        print(actual)
        self.assertEqual(len(expected), len(actual))
        self.assertEqual(None, cube.time.chunks)
        np.testing.assert_equal(actual, expected)

    def test_replace_time_slice(self):
        self.write_cube('2019-01-02', 10)

        replace_time_slice(self.CUBE_PATH, 5, self.make_slice('2019-01-06T02:00'))
        replace_time_slice(self.CUBE_PATH, 9, self.make_slice('2019-01-11T02:00'))
        replace_time_slice(self.CUBE_PATH, 0, self.make_slice('2019-01-01T02:00'))

        cube = xr.open_zarr(self.CUBE_PATH)
        expected = np.array(['2019-01-01T14:00', '2019-01-03T12:00',
                             '2019-01-04T12:00', '2019-01-05T12:00',
                             '2019-01-06T12:00', '2019-01-06T14:00',
                             '2019-01-08T12:00', '2019-01-09T12:00',
                             '2019-01-10T12:00', '2019-01-11T14:00'], dtype=cube.time.dtype)
        self.assertEqual(10, cube.time.size)
        self.assertEqual(None, cube.time.chunks)
        np.testing.assert_equal(cube.time.values, expected)

    def test_update_corrupt_cube(self):
        self.write_cube('2019-01-01', 3)

        cube = xr.open_zarr(self.CUBE_PATH)
        t, y, x = cube.precipitation.shape
        new_shape = y, t, x
        t, y, x = cube.precipitation.dims
        new_dims = y, t, x
        cube['precipitation'] = xr.DataArray(cube.precipitation.values.reshape(new_shape),
                                             dims=new_dims,
                                             coords=cube.precipitation.coords)
        cube.to_zarr(self.CUBE_PATH_2)

        with self.assertRaises(ValueError) as cm:
            insert_time_slice(self.CUBE_PATH_2, 2, self.make_slice('2019-01-02T06:30'))
        self.assertEqual("dimension 'time' of variable 'precipitation' must be first dimension",
                         f"{cm.exception}")


class ZarrStoreTest(unittest.TestCase):
    CUBE_PATH = 'store-test-cube.zarr'

    def setUp(self) -> None:
        rimraf(self.CUBE_PATH)

    def tearDown(self) -> None:
        rimraf(self.CUBE_PATH)

    def test_local(self):
        cube = new_cube(time_periods=10, time_start='2019-01-01',
                        variables=dict(precipitation=0.1,
                                       temperature=270.5,
                                       soil_moisture=0.2))
        cube = chunk_dataset(cube, dict(time=1, lat=90, lon=90), format_name='zarr')
        cube.to_zarr(self.CUBE_PATH)
        cube.close()

        diagnostic_store = DiagnosticStore(zarr.DirectoryStore(self.CUBE_PATH),
                                           logging_observer(log_path='local-cube.log'))
        xr.open_zarr(diagnostic_store)

    @unittest.skipUnless(False, 'is enabled')
    def test_remote(self):
        import s3fs
        endpoint_url = "https://s3.eu-central-1.amazonaws.com"
        s3 = s3fs.S3FileSystem(anon=True, client_kwargs=dict(endpoint_url=endpoint_url))
        s3_store = s3fs.S3Map(root="xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr", s3=s3, check=False)
        diagnostic_store = DiagnosticStore(s3_store, logging_observer(log_path='remote-cube.log'))
        xr.open_zarr(diagnostic_store)


# test/core/test_unchunk.py

import os.path
import unittest

from xcube.constants import FORMAT_NAME_ZARR
from xcube.core.chunk import chunk_dataset
from xcube.core.dsio import rimraf
from xcube.core.new import new_cube
from xcube.core.unchunk import unchunk_dataset


class UnchunkDatasetTest(unittest.TestCase):
    TEST_ZARR = 'test.zarr'

    def setUp(self):
        rimraf(self.TEST_ZARR)
        cube = new_cube(variables=dict(A=0.5, B=-1.5))
        cube = chunk_dataset(cube, chunk_sizes=dict(time=1, lat=90, lon=90), format_name=FORMAT_NAME_ZARR)
        cube.to_zarr(self.TEST_ZARR)

        self.chunked_a_files = {'.zarray',
                                '.zattrs',
                                '0.0.0', '0.0.1', '0.0.2', '0.0.3', '0.1.0', '0.1.1', '0.1.2', '0.1.3',
                                '1.0.0', '1.0.1', '1.0.2', '1.0.3', '1.1.0', '1.1.1', '1.1.2', '1.1.3',
                                '2.0.0', '2.0.1', '2.0.2', '2.0.3', '2.1.0', '2.1.1', '2.1.2', '2.1.3',
                                '3.0.0', '3.0.1', '3.0.2', '3.0.3', '3.1.0', '3.1.1', '3.1.2', '3.1.3',
                                '4.0.0', '4.0.1', '4.0.2', '4.0.3', '4.1.0', '4.1.1', '4.1.2', '4.1.3'}
        self.chunked_b_files = self.chunked_a_files
        self.chunked_time_files = {'.zarray', '.zattrs', '0', '1', '2', '3', '4'}
        self.chunked_lat_files = {'.zattrs', '.zarray', '0', '1'}
        self.chunked_lon_files = {'.zattrs', '.zarray', '0', '1', '2', '3'}

    def tearDown(self):
        rimraf(self.TEST_ZARR)

    def test_chunked(self):
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files=self.chunked_b_files,
                                expected_time_files=self.chunked_time_files,
                                expected_lat_files=self.chunked_lat_files,
                                expected_lon_files=self.chunked_lon_files)

    def test_unchunk_all(self):
        unchunk_dataset(self.TEST_ZARR)
        self._assert_cube_files(expected_a_files={'.zarray', '.zattrs', '0.0.0'},
                                expected_b_files={'.zarray', '.zattrs', '0.0.0'},
                                expected_time_files={'.zarray', '.zattrs', '0'},
                                expected_lat_files={'.zarray', '.zattrs', '0'},
                                expected_lon_files={'.zarray', '.zattrs', '0'})

    def test_unchunk_coords_only(self):
        unchunk_dataset(self.TEST_ZARR, coords_only=True)
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files=self.chunked_b_files,
                                expected_time_files={'.zarray', '.zattrs', '0'},
                                expected_lat_files={'.zarray', '.zattrs', '0'},
                                expected_lon_files={'.zarray', '.zattrs', '0'})

    def test_unchunk_data_var(self):
        unchunk_dataset(self.TEST_ZARR, var_names=['B'])
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files={'.zarray', '.zattrs', '0.0.0'},
                                expected_time_files=self.chunked_time_files,
                                expected_lat_files=self.chunked_lat_files,
                                expected_lon_files=self.chunked_lon_files)

    def test_unchunk_coord_var(self):
        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files=self.chunked_b_files,
                                expected_time_files={'.zarray', '.zattrs', '0'},
                                expected_lat_files=self.chunked_lat_files,
                                expected_lon_files=self.chunked_lon_files)

    def test_dont_unchunk_if_unchunked(self):
        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
        unchunk_dataset(self.TEST_ZARR, var_names=['time'], coords_only=True)
        self._assert_cube_files(expected_a_files=self.chunked_a_files,
                                expected_b_files=self.chunked_b_files,
                                expected_time_files={'.zarray', '.zattrs', '0'},
                                expected_lat_files=self.chunked_lat_files,
                                expected_lon_files=self.chunked_lon_files)

    def test_unchunk_data_var_coords_only(self):
        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR, var_names=['B'], coords_only=True)
        self.assertEqual("variable 'B' is not a coordinate variable in 'test.zarr'", f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR, var_names=['C'], coords_only=False)
        self.assertEqual("variable 'C' is not a variable in 'test.zarr'", f'{cm.exception}')

    def test_unchunk_invalid_path(self):
        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR + '.zip')
        self.assertEqual("'test.zarr.zip' is not a valid Zarr directory", f'{cm.exception}')

    def test_unchunk_invalid_vars(self):
        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR, var_names=['times'], coords_only=True)
        self.assertEqual("variable 'times' is not a coordinate variable in 'test.zarr'", f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            unchunk_dataset(self.TEST_ZARR, var_names=['CHL'], coords_only=False)
        self.assertEqual("variable 'CHL' is not a variable in 'test.zarr'", f'{cm.exception}')

    def _assert_cube_files(self,
                           expected_a_files, expected_b_files,
                           expected_time_files, expected_lat_files, expected_lon_files):
        self._assert_files('A', expected_a_files)
        self._assert_files('B', expected_b_files)
        self._assert_files('time', expected_time_files)
        self._assert_files('lat', expected_lat_files)
        self._assert_files('lon', expected_lon_files)

    def _assert_files(self, var_name, expected_files):
        actual_files = os.listdir(os.path.join(self.TEST_ZARR, var_name))
        self.assertEqual(set(expected_files), set(actual_files))


# test/core/test_update.py

import unittest

import numpy as np
import pandas as pd
import xarray as xr

from test.sampledata import create_highroc_dataset
from xcube.core.update import update_dataset_var_attrs, update_dataset_attrs


class UpdateVariablePropsTest(unittest.TestCase):
    def test_no_change(self):
        ds1 = create_highroc_dataset()
        # noinspection PyTypeChecker
        ds2 = update_dataset_var_attrs(ds1, None)
        self.assertIs(ds2, ds1)
        ds2 = update_dataset_var_attrs(ds1, [])
        self.assertIs(ds2, ds1)

    def test_change_all_or_none(self):
        ds1 = create_highroc_dataset()
        ds2 = update_dataset_var_attrs(ds1,
                                       [(var_name, {'marker': True}) for var_name in ds1.data_vars])
        self.assertEqual(len(ds1.data_vars), len(ds2.data_vars))
        self.assertTrue(all(['marker' in ds2[n].attrs for n in ds2.variables]))

        with self.assertRaises(KeyError):
            update_dataset_var_attrs(ds1, [('bibo', {'marker': True})])

    def test_change_some(self):
        ds1 = create_highroc_dataset()
        ds2 = update_dataset_var_attrs(ds1,
                                       [('conc_chl', {'name': 'chl_c2rcc'}),
                                        ('c2rcc_flags', {'name': 'flags', 'marker': True}),
                                        ('rtoa_10', None)])

        self.assertEqual(len(ds1.data_vars), len(ds2.data_vars))

        self.assertNotIn('conc_chl', ds2.data_vars)
        self.assertNotIn('c2rcc_flags', ds2.data_vars)

        self.assertIn('chl_c2rcc', ds2.data_vars)
        self.assertIn('original_name', ds2.chl_c2rcc.attrs)
        self.assertEqual('conc_chl', ds2.chl_c2rcc.attrs['original_name'])

        self.assertIn('flags', ds2.data_vars)
        self.assertIn('original_name', ds2.flags.attrs)
        self.assertEqual('c2rcc_flags', ds2.flags.attrs['original_name'])
        self.assertIn('marker', ds2.flags.attrs)
        self.assertEqual(True, ds2.flags.attrs['marker'])

        self.assertIn('rtoa_10', ds2.data_vars)

        with self.assertRaises(ValueError) as cm:
            update_dataset_var_attrs(ds1, [('conc_chl', None),
                                           ('c2rcc_flags', None),
                                           ('rtoa_1', {'name': 'refl_toa'}),
                                           ('rtoa_2', {'name': 'refl_toa'}),
                                           ('rtoa_3', {'name': 'refl_toa'})])
        self.assertEqual("variable 'rtoa_2' cannot be renamed into 'refl_toa' because the name is already in use",
                         f'{cm.exception}')


class UpdateGlobalAttributesTest(unittest.TestCase):

    def test_update_global_attributes(self):
        num_lons = 8
        num_lats = 6
        num_times = 5

        lon_min = -20.
        lat_min = 12.

        res = 0.25
        res05 = res / 2
        lon = np.linspace(lon_min + res05, lon_min + num_lons * res - res05, num_lons)
        lat = np.linspace(lat_min + res05, lat_min + num_lats * res - res05, num_lats)
        lon_bnds = np.array([[v - res05, v + res05] for v in lon])
        lat_bnds = np.array([[v - res05, v + res05] for v in lat])
        time = [pd.to_datetime(f'2018-06-0{i}T12:00:00') for i in range(1, num_times + 1)]
        time_bnds = [(pd.to_datetime(f'2018-06-0{i}T00:00:00'),
                      pd.to_datetime(f'2018-06-0{i}T23:00:59')) for i in range(1, num_times + 1)]

        coords = dict(time=(['time'], time),
                      lat=(['lat'], lat),
                      lon=(['lon'], lon))

        coords_with_bnds = dict(time_bnds=(['time', 'bnds'], time_bnds),
                                lat_bnds=(['lat', 'bnds'], lat_bnds),
                                lon_bnds=(['lon', 'bnds'], lon_bnds),
                                **coords)

        output_metadata = dict(history='pipo', license='MIT', Conventions='CF-1.7')

        ds1 = xr.Dataset(coords=coords)
        ds2 = update_dataset_attrs(ds1, global_attrs=output_metadata)

        self.assertIsNot(ds2, ds1)
        self.assertEqual('CF-1.7', ds2.attrs.get('Conventions'))
        self.assertEqual('MIT', ds2.attrs.get('license'))
        self.assertEqual('pipo', ds2.attrs.get('history'))
        self.assertEqual(-20.0, ds2.attrs.get('geospatial_lon_min'))
        self.assertEqual(-18.0, ds2.attrs.get('geospatial_lon_max'))
        self.assertEqual(0.25, ds2.attrs.get('geospatial_lon_resolution'))
        self.assertEqual('degrees_east', ds2.attrs.get('geospatial_lon_units'))
        self.assertEqual(12.0, ds2.attrs.get('geospatial_lat_min'))
        self.assertEqual(13.5, ds2.attrs.get('geospatial_lat_max'))
        self.assertEqual(0.25, ds2.attrs.get('geospatial_lat_resolution'))
        self.assertEqual('degrees_north', ds2.attrs.get('geospatial_lat_units'))
        self.assertEqual('2018-06-01T00:00:00.000000000', ds2.attrs.get('time_coverage_start'))
        self.assertEqual('2018-06-06T00:00:00.000000000', ds2.attrs.get('time_coverage_end'))
        self.assertIn('date_modified', ds2.attrs)

        ds1 = xr.Dataset(coords=coords_with_bnds)
        ds2 = update_dataset_attrs(ds1, global_attrs=output_metadata)

        self.assertIsNot(ds2, ds1)
        self.assertEqual('CF-1.7', ds2.attrs.get('Conventions'))
        self.assertEqual('MIT', ds2.attrs.get('license'))
        self.assertEqual('pipo', ds2.attrs.get('history'))
        self.assertEqual(-20.0, ds2.attrs.get('geospatial_lon_min'))
        self.assertEqual(-18.0, ds2.attrs.get('geospatial_lon_max'))
        self.assertEqual(0.25, ds2.attrs.get('geospatial_lon_resolution'))
        self.assertEqual('degrees_east', ds2.attrs.get('geospatial_lon_units'))
        self.assertEqual(12.0, ds2.attrs.get('geospatial_lat_min'))
        self.assertEqual(13.5, ds2.attrs.get('geospatial_lat_max'))
        self.assertEqual(0.25, ds2.attrs.get('geospatial_lat_resolution'))
        self.assertEqual('degrees_north', ds2.attrs.get('geospatial_lat_units'))
        self.assertEqual('2018-06-01T00:00:00.000000000', ds2.attrs.get('time_coverage_start'))
        self.assertEqual('2018-06-05T23:00:59.000000000', ds2.attrs.get('time_coverage_end'))
        self.assertIn('date_modified', ds2.attrs)


# test/core/test_vars2dim.py

import unittest

from xcube.core.new import new_cube
from xcube.core.vars2dim import vars_to_dim


class VarsToDimTest(unittest.TestCase):
    def test_vars_to_dim(self):
        dataset = new_cube(variables=dict(precipitation=0.4, temperature=275.2))

        ds = vars_to_dim(dataset)

        self.assertIn("var", ds.dims)
        self.assertEqual(2, ds.dims["var"])
        self.assertIn("var", ds.coords)
        self.assertIn("data", ds.data_vars)
        var_names = ds["var"]
        self.assertEqual(("var",), var_names.dims)
        self.assertTrue(hasattr(var_names, "encoding"))
        self.assertEqual(2, len(var_names))
        self.assertIn("precipitation", str(var_names[0]))
        self.assertIn("temperature", str(var_names[1]))


# test/core/test_verify.py

import unittest

import numpy as np
import xarray as xr

from xcube.core.new import new_cube
from xcube.core.verify import verify_cube, assert_cube


class AssertAndVerifyCubeTest(unittest.TestCase):

    def test_assert_cube_ok(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        self.assertIs(cube, assert_cube(cube))

    def test_assert_cube_without_bounds(self):
        cube = new_cube(variables=dict(precipitation=0.5), drop_bounds=True)
        self.assertIs(cube, assert_cube(cube))

    def test_assert_cube_illegal_coord_var(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        cube = cube.assign_coords(lat=xr.DataArray(np.outer(cube.lat, np.ones(cube.lon.size)),
                                                   dims=("y", "x")),
                                  lon=xr.DataArray(np.outer(np.ones(cube.lat.size), cube.lon),
                                                   dims=("y", "x")))
        with self.assertRaises(ValueError) as cm:
            assert_cube(cube)
        self.assertEqual("Dataset is not a valid xcube dataset, because:\n"
                         "- missing spatial x,y coordinate variables.",
                         f"{cm.exception}")

    def test_assert_cube_illegal_coord_bounds_var(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        lat_bnds = np.zeros((cube.time.size, cube.lat.size, 2))
        lon_bnds = np.zeros((cube.time.size, cube.lon.size, 2), dtype=np.float16)
        lat_bnds[:, :, :] = cube.lat_bnds
        lon_bnds[:, :, :] = cube.lon_bnds
        cube = cube.assign_coords(lat_bnds=xr.DataArray(lat_bnds, dims=("time", "lat", "bnds")),
                                  lon_bnds=xr.DataArray(lon_bnds, dims=("time", "lon", "bnds")))
        with self.assertRaises(ValueError) as cm:
            assert_cube(cube)
        self.assertEqual("Dataset is not a valid xcube dataset, because:\n"
                         "- bounds coordinate variable 'lon_bnds' must have dimensions ('lon', <bounds_dim>);\n"
                         "- shape of bounds coordinate variable 'lon_bnds' must be (360, 2) but was (5, 360, 2);\n"
                         "- type of bounds coordinate variable 'lon_bnds' must be dtype('float64') but was dtype('float16');\n"
                         "- bounds coordinate variable 'lat_bnds' must have dimensions ('lat', <bounds_dim>);\n"
                         "- shape of bounds coordinate variable 'lat_bnds' must be (180, 2) but was (5, 180, 2).",
                         f"{cm.exception}")

    def test_assert_cube_illegal_data_var(self):
        cube = new_cube(variables=dict(precipitation=0.5))
        shape = cube.dims["lat"], cube.dims["lon"]
        cube["chl"] = xr.DataArray(np.random.rand(*shape),
                                   dims=("lat", "lon"),
                                   coords=dict(lat=cube.lat, lon=cube.lon))
        with self.assertRaises(ValueError) as cm:
            assert_cube(cube)
        self.assertEqual("Dataset is not a valid xcube dataset, because:\n"
                         "- dimensions of data variable 'chl' must be"
                         " ('time', ..., 'lat', 'lon'), but were ('lat', 'lon') for 'chl';\n"
                         "- dimensions of all data variables must be same,"
                         " but found ('time', 'lat', 'lon') for 'precipitation'"
                         " and ('lat', 'lon') for 'chl'.",
                         f"{cm.exception}")

    def test_verify_cube(self):
        cube = new_cube()
        self.assertEqual([], verify_cube(cube))
        ds = cube.drop("time")
        self.assertEqual(["missing time coordinate variable"], verify_cube(ds))
        ds = ds.drop("lat")
        self.assertEqual(["missing spatial x,y coordinate variables",
                          "missing time coordinate variable"], verify_cube(ds))

    def test_verify_cube_coord_equidistance(self):
        ds = new_cube()
        ds['lat'] = ds['lat'] + np.random.rand(len(ds['lat']))
        ds['lat_bnds'][:, 0] = ds['lat_bnds'][:, 0] + np.random.rand(len(ds['lat_bnds'][:, 0]))
        ds['lat_bnds'][:, 1] = ds['lat_bnds'][:, 1] + np.random.rand(len(ds['lat_bnds'][:, 1]))

        self.assertEqual(["coordinate variable 'lat' is not equidistant",
                          "coordinate variable 'lat_bnds' is not equidistant"],
                         verify_cube(ds))

    def test_verify_cube_coord_equidistance_reverse_lat(self):
        ds = new_cube()
        ds['lat'] = np.flip(ds.lat)
        ds['lat_bnds'] = xr.DataArray(np.flip(ds.lat_bnds.values),
                                      dims=['lat', 'bnds'])

        ds['lon'] = np.roll(ds.lon, 90)
        ds['lon_bnds'] = xr.DataArray(np.roll(ds.lon_bnds.values, 180),
                                      dims=['lon', 'bnds'])

        self.assertEqual([], verify_cube(ds))


# test/core/test_xarray.py

import unittest

import xarray as xr

from test.sampledata import new_test_dataset
from xcube.core.gridmapping import GridMapping
from xcube.core.new import new_cube
from xcube.core.xarray import DatasetAccessor


class XCubeDatasetAccessorTest(unittest.TestCase):
    # noinspection PyMethodMayBeStatic
    def test_init(self):
        DatasetAccessor(xr.Dataset())

    def test_installed(self):
        self.assertTrue(hasattr(xr.Dataset, "xcube"))
        ds = xr.Dataset()
        self.assertTrue(hasattr(ds, "xcube"))

    def test_non_empty_cube_subset(self):
        dataset = new_cube(variables=dict(a=9, b=0.2))
        cube = dataset.xcube.cube
        self.assertIsInstance(cube, xr.Dataset)
        self.assertEqual(set(dataset.data_vars), set(cube.data_vars))
        gm = dataset.xcube.gm
        self.assertIsInstance(gm, GridMapping)

    def test_no_cube_subset(self):
        dataset = xr.Dataset(dict(a=9, b=0.2))
        cube = dataset.xcube.cube
        self.assertIsInstance(cube, xr.Dataset)
        self.assertEqual(set(), set(cube.data_vars))
        self.assertIs(None, dataset.xcube.gm)
        self.assertIs(dataset, dataset.xcube.non_cube)

    ########################################################################
    # Testing old API from here on.
    #
    # Let's quickly agree, if we should deprecate all this stuff. I guess,
    # no one uses it.
    #
    # We should only add props and methods to this accessor
    # that require a certain state to be hold.
    # Such state could be props that are expensive to recompute,
    # such as grid mappings.
    #
    # It causes too much overhead and maintenance work
    # if we continue putting any xcube function here.
    ########################################################################

    def test_new(self):
        self.assertIsInstance(xr.Dataset.xcube.new(), xr.Dataset)

    def test_select_vars(self):
        self.assertIsInstance(xr.Dataset().xcube.select_variables_subset(), xr.Dataset)

    def test_dump(self):
        self.assertIsInstance(xr.Dataset().xcube.dump(), str)

    def test_vars_to_dim(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05"],
                                   precipitation=0.4, temperature=275.2)

        self.assertIsInstance(dataset.xcube.vars_to_dim(), xr.Dataset)

    def test_levels(self):
        dataset = new_test_dataset(["2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05"],
                                   precipitation=0.4, temperature=275.2)
        levels = dataset.xcube.levels(spatial_tile_shape=(45, 45))
        self.assertIsInstance(levels, list)
        self.assertEqual(3, len(levels))
        self.assertTrue(all(isinstance(level, xr.Dataset) for level in levels))
        self.assertTrue(all("precipitation" in level for level in levels))
        self.assertTrue(all("temperature" in level for level in levels))
        self.assertEqual([(5, 180, 360), (5, 90, 180), (5, 45, 90)],
                         [level.precipitation.shape for level in levels])
        self.assertEqual([((1, 1, 1, 1, 1), (45, 45, 45, 45), (45, 45, 45, 45, 45, 45, 45, 45)),
                          ((1, 1, 1, 1, 1), (45, 45), (45, 45, 45, 45)),
                          ((1, 1, 1, 1, 1), (45,), (45, 45))],
                         [level.precipitation.chunks for level in levels])


# test/server/mocks.py

#  The MIT License (MIT)
#  Copyright (c) 2022 by the xcube development team and contributors
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.

from typing import Optional, Sequence, Tuple, Dict, Any, Union, Callable, \
    Awaitable, List, Mapping

from tornado import concurrent

from xcube.constants import EXTENSION_POINT_SERVER_APIS
from xcube.server.api import Api
from xcube.server.api import ApiContext
from xcube.server.api import ApiRequest
from xcube.server.api import ApiResponse
from xcube.server.api import ApiRoute
from xcube.server.api import Context
from xcube.server.api import JSON
from xcube.server.api import ReturnT
from xcube.server.api import ServerConfig
from xcube.server.framework import Framework
from xcube.server.server import Server
from xcube.util.extension import ExtensionRegistry

ApiSpec = Union[Api,
                str,
                Tuple[str, Dict[str, Any]]]

ApiSpecs = Sequence[ApiSpec]


def mock_server(framework: Optional[Framework] = None,
                config: Optional[ServerConfig] = None,
                api_specs: Optional[ApiSpecs] = None) -> Server:
    return Server(
        framework or MockFramework(),
        config or {},
        extension_registry=mock_extension_registry(api_specs or ())
    )


def mock_extension_registry(api_specs: ApiSpecs) -> ExtensionRegistry:
    extension_registry = ExtensionRegistry()
    for api in api_specs:
        if isinstance(api, str):
            api_name = api
            api = Api(api_name)
        elif not isinstance(api, Api):
            api_name, api_kwargs = api
            api = Api(api_name, **api_kwargs)
        extension_registry.add_extension(EXTENSION_POINT_SERVER_APIS,
                                         api.name,
                                         component=api)
    return extension_registry


class MockFramework(Framework):

    def __init__(self):
        self.add_routes_count = 0
        self.update_count = 0
        self.start_count = 0
        self.stop_count = 0
        self.call_later_count = 0
        self.run_in_executor_count = 0

    def add_routes(self, routes: Sequence[ApiRoute]):
        self.add_routes_count += 1

    def update(self, ctx: Context):
        self.update_count += 1

    def start(self, ctx: Context):
        self.start_count += 1

    def stop(self, ctx: Context):
        self.stop_count += 1

    def call_later(self,
                   delay: Union[int, float],
                   callback: Callable,
                   *args,
                   **kwargs) -> object:
        self.call_later_count += 1
        return object()

    def run_in_executor(self,
                        executor: Optional[concurrent.futures.Executor],
                        function: Callable[..., ReturnT],
                        *args: Any,
                        **kwargs: Any) -> Awaitable[ReturnT]:
        self.run_in_executor_count += 1
        import concurrent.futures
        # noinspection PyTypeChecker
        return concurrent.futures.Future()


class MockApiContext(ApiContext):
    def __init__(self, server_ctx: Context):
        super().__init__(server_ctx)
        self.on_update_count = 0
        self.on_dispose_count = 0

    def on_update(self, prev_ctx: Optional[ApiContext]):
        self.on_update_count += 1

    def on_dispose(self):
        self.on_dispose_count += 1


class MockApiRequest(ApiRequest):

    def __init__(self,
                 query_args: Optional[Mapping[str, Sequence[str]]] = None):
        self._query_args = query_args or {}

    @property
    def query(self) -> Mapping[str, Sequence[str]]:
        return self._query_args

    # noinspection PyShadowingBuiltins
    def get_query_args(self, name: str, type: Any = None) -> Sequence[Any]:
        args = self._query_args.get(name, [])
        return [type(arg) for arg in args] if type is not None else args

    def url_for_path(self, path: str, query: Optional[str] = None) -> str:
        return ''

    @property
    def headers(self) -> Mapping[str, str]:
        return {}

    @property
    def url(self) -> str:
        return ''

    @property
    def body(self) -> bytes:
        return bytes({})

    @property
    def json(self) -> JSON:
        return {}



class MockApiResponse(ApiResponse):
    def set_header(self, name: str, value: str):
        pass

    def set_status(self, status_code: int, reason: Optional[str] = None):
        pass

    def write(self, data: Union[str, bytes, JSON]):
        pass

    def finish(self, data: Union[str, bytes, JSON] = None):
        pass


# test/server/test_api.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest
from typing import Optional

from xcube.server.api import Api
from xcube.server.api import ApiContext
from xcube.server.api import ApiHandler
from xcube.server.api import ApiRoute
from xcube.server.api import Context
from xcube.server.api import ApiError
from xcube.server.server import ServerContext
from .mocks import MockApiRequest
from .mocks import MockApiResponse
from .mocks import MockFramework
from .mocks import mock_server


class ApiTest(unittest.TestCase):

    def test_basic_props(self):
        api = Api("datasets", description='What an API!')
        self.assertEqual("datasets", api.name)
        self.assertEqual("0.0.0", api.version)
        self.assertEqual('What an API!', api.description)
        self.assertEqual((), api.required_apis)
        self.assertEqual((), api.optional_apis)
        self.assertEqual(None, api.config_schema)
        self.assertEqual((), api.routes)

    def test_ctor_functions(self):
        class MyApiContext(ApiContext):
            def on_update(self, prev_ctx: Optional["Context"]):
                pass

        test_dict = dict()

        def handle_start(ctx):
            test_dict['handle_start'] = ctx

        def handle_stop(ctx):
            test_dict['handle_stop'] = ctx

        server_ctx = ServerContext(mock_server(), {})

        api = Api("datasets",
                  create_ctx=MyApiContext,
                  on_start=handle_start,
                  on_stop=handle_stop)

        api_ctx = api.create_ctx(server_ctx)
        self.assertIsInstance(api_ctx, MyApiContext)

        api.on_start(server_ctx)
        self.assertIs(server_ctx, test_dict.get('handle_start'))
        self.assertIs(None, test_dict.get('handle_stop'))

        api.on_stop(server_ctx)
        self.assertIs(server_ctx, test_dict.get('handle_start'))
        self.assertIs(server_ctx, test_dict.get('handle_stop'))

    def test_route_decorator(self):
        api = Api("datasets")

        @api.route("/datasets")
        class DatasetsHandler(ApiHandler):
            # noinspection PyMethodMayBeStatic
            def get(self):
                return {}

        @api.route("/datasets/{dataset_id}")
        class DatasetHandler(ApiHandler):
            # noinspection PyMethodMayBeStatic
            def get(self):
                return {}

        self.assertEqual(
            (
                ApiRoute("datasets", "/datasets", DatasetsHandler),
                ApiRoute("datasets", "/datasets/{dataset_id}", DatasetHandler)
            ),
            api.routes
        )

    def test_operation_decorator(self):
        api = Api("datasets")

        @api.route("/datasets")
        class DatasetsHandler(ApiHandler):
            # noinspection PyMethodMayBeStatic
            @api.operation()
            def get(self):
                return {}

        self.assertEqual(
            {},
            getattr(DatasetsHandler.get, '__openapi__', None)
        )

        with self.assertRaises(TypeError) as cm:
            api.operation()(42)
        self.assertEqual("API datasets: @operation() decorator"
                         " must be used with one of the HTTP"
                         " methods of an ApiHandler",
                         f"{cm.exception}")


class ApiRouteTest(unittest.TestCase):

    def test_equal(self):
        class DatasetHandler(ApiHandler):
            # noinspection PyMethodMayBeStatic
            def get(self):
                return {}

        self.assertTrue(
            ApiRoute("datasets", "/datasets", DatasetHandler) ==
            ApiRoute("datasets", "/datasets", DatasetHandler)
        )
        self.assertTrue(
            ApiRoute("datasets", "/datasets", DatasetHandler) ==
            ApiRoute("datasets", "/datasets", DatasetHandler, {})
        )
        self.assertTrue(
            ApiRoute("datasets", "/datasets", DatasetHandler) !=
            ApiRoute("datasets", "/dataset", DatasetHandler, {})
        )
        self.assertTrue(
            ApiRoute("datasets", "/datasets", DatasetHandler) !=
            ApiRoute("dataset", "/datasets", DatasetHandler, {})
        )
        self.assertTrue(
            ApiRoute("datasets", "/datasets", DatasetHandler) != 42
        )

    def test_hash(self):
        class DatasetHandler(ApiHandler):
            # noinspection PyMethodMayBeStatic
            def get(self):
                return {}

        self.assertEqual(
            hash(ApiRoute("datasets", "/datasets", DatasetHandler)),
            hash(ApiRoute("datasets", "/datasets", DatasetHandler))
        )
        self.assertEqual(
            hash(ApiRoute("datasets", "/datasets", DatasetHandler)),
            hash(ApiRoute("datasets", "/datasets", DatasetHandler, {}))
        )

    def test_str_and_repr(self):
        class DatasetHandler(ApiHandler):
            # noinspection PyMethodMayBeStatic
            def get(self):
                return {}

        api_route = ApiRoute("datasets", "/datasets",
                             DatasetHandler, dict(force=True))
        self.assertEqual("ApiRoute("
                         "'datasets',"
                         " '/datasets',"
                         " DatasetHandler,"
                         " handler_kwargs={'force': True}"
                         ")",
                         f'{api_route}')
        self.assertEqual("ApiRoute("
                         "'datasets',"
                         " '/datasets',"
                         " DatasetHandler,"
                         " handler_kwargs={'force': True}"
                         ")",
                         f'{api_route!r}')


class ApiContextTest(unittest.TestCase):
    def test_basic_props(self):
        config = {}
        server_ctx = ServerContext(mock_server(), config)
        api_ctx = ApiContext(server_ctx)
        self.assertIs(config, api_ctx.config)
        self.assertEqual((), api_ctx.apis)
        self.assertIsInstance(api_ctx.open_api_doc, dict)

    def test_async_exec(self):
        framework = MockFramework()
        server = mock_server(framework=framework)
        server_ctx = ServerContext(server, {})
        api_ctx = ApiContext(server_ctx)

        def my_func(a, b):
            return a + b

        self.assertEqual(0, framework.call_later_count)
        api_ctx.call_later(0.1, my_func, 40, 2)
        self.assertEqual(1, framework.call_later_count)

        self.assertEqual(0, framework.run_in_executor_count)
        api_ctx.run_in_executor(None, my_func, 40, 2)
        self.assertEqual(1, framework.run_in_executor_count)


class ApiHandlerTest(unittest.TestCase):
    class DatasetsContext(ApiContext):

        def on_update(self, prev_ctx: Optional[Context]):
            pass

    def setUp(self) -> None:
        self.api = Api("datasets", create_ctx=self.DatasetsContext)
        self.config = {}
        server_ctx = ServerContext(mock_server(api_specs=[self.api]),
                                   self.config)
        server_ctx.on_update(None)
        self.request = MockApiRequest()
        self.response = MockApiResponse()
        self.handler = ApiHandler(server_ctx.get_api_ctx("datasets"),
                                  self.request,
                                  self.response)

    def test_props(self):
        handler = self.handler
        self.assertIs(self.request, handler.request)
        self.assertIs(self.response, handler.response)
        self.assertIsInstance(handler.ctx, self.DatasetsContext)
        self.assertIs(self.config, handler.ctx.config)

    def test_default_methods(self):
        handler = self.handler

        with self.assertRaises(ApiError.MethodNotAllowed) as cm:
            handler.get()
        self.assertEqual('HTTP status 405: method GET not allowed',
                         f'{cm.exception}')

        with self.assertRaises(ApiError.MethodNotAllowed) as cm:
            handler.post()
        self.assertEqual('HTTP status 405: method POST not allowed',
                         f'{cm.exception}')

        with self.assertRaises(ApiError.MethodNotAllowed) as cm:
            handler.put()
        self.assertEqual('HTTP status 405: method PUT not allowed',
                         f'{cm.exception}')

        with self.assertRaises(ApiError.MethodNotAllowed) as cm:
            handler.delete()
        self.assertEqual('HTTP status 405: method DELETE not allowed',
                         f'{cm.exception}')

        with self.assertRaises(ApiError.MethodNotAllowed) as cm:
            handler.options()
        self.assertEqual('HTTP status 405: method OPTIONS not allowed',
                         f'{cm.exception}')


class ApiRequestTest(unittest.TestCase):

    def test_query_args(self):
        request = MockApiRequest(query_args=dict(details=['1']))
        self.assertEqual(['1'], request.get_query_args('details'))
        self.assertEqual([], request.get_query_args('crs'))

    def test_query_args_with_type(self):
        request = MockApiRequest(query_args=dict(details=['1']))
        self.assertEqual([True], request.get_query_args('details', type=bool))
        self.assertEqual([], request.get_query_args('crs', type=str))

    def test_query_arg_with_type(self):
        request = MockApiRequest(query_args=dict(details=['1']))
        self.assertEqual(True, request.get_query_arg('details', type=bool))
        self.assertEqual(None, request.get_query_arg('crs', type=str))

    def test_query_arg_with_default(self):
        request = MockApiRequest(query_args=dict(details=['1']))
        self.assertEqual(True, request.get_query_arg('details',
                                                     default=False))
        self.assertEqual('CRS84', request.get_query_arg('crs',
                                                        default='CRS84'))


class ApiErrorTest(unittest.TestCase):

    def test_base_class(self):
        error = ApiError(428)
        self.assertIsInstance(error, Exception)
        self.assertEqual(428, error.status_code)
        self.assertEqual(None, error.message)

        error = ApiError(428, message='No idea')
        self.assertIsInstance(error, Exception)
        self.assertEqual(428, error.status_code)
        self.assertEqual('No idea', error.message)

        self.assertIsNot(ApiError.BadRequest, ApiError.Unauthorized)
        self.assertIsNot(ApiError.BadRequest, ApiError.Forbidden)

    def test_bad_request(self):
        error = ApiError.BadRequest()
        self.assertIsInstance(error, ApiError)
        self.assertEqual(400, error.status_code)

    def test_unauthorized(self):
        error = ApiError.Unauthorized()
        self.assertIsInstance(error, ApiError)
        self.assertEqual(401, error.status_code)

    def test_forbidden(self):
        error = ApiError.Forbidden()
        self.assertIsInstance(error, ApiError)
        self.assertEqual(403, error.status_code)

    def test_not_found(self):
        error = ApiError.NotFound()
        self.assertIsInstance(error, ApiError)
        self.assertEqual(404, error.status_code)

    def test_method_not_allowed(self):
        error = ApiError.MethodNotAllowed()
        self.assertIsInstance(error, ApiError)
        self.assertEqual(405, error.status_code)

    def test_conflict(self):
        error = ApiError.Conflict()
        self.assertIsInstance(error, ApiError)
        self.assertEqual(409, error.status_code)

    def test_gone(self):
        error = ApiError.Gone()
        self.assertIsInstance(error, ApiError)
        self.assertEqual(410, error.status_code)

    def test_not_implemented(self):
        error = ApiError.NotImplemented()
        self.assertIsInstance(error, ApiError)
        self.assertEqual(501, error.status_code)

    def test_invalid_server_config(self):
        error = ApiError.InvalidServerConfig()
        self.assertIsInstance(error, ApiError)
        self.assertEqual(580, error.status_code)


# test/server/test_framework.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest

from xcube.plugin import init_plugin
from xcube.server.framework import Framework
from xcube.server.framework import get_framework_class
from xcube.server.framework import get_framework_names
from xcube.util.plugin import get_extension_registry


class FrameworkTest(unittest.TestCase):

    @classmethod
    def setUpClass(cls) -> None:
        extension_registry = get_extension_registry()
        init_plugin(extension_registry)

    def test_get_framework_names(self):
        framework_names = get_framework_names()
        self.assertEqual({"tornado", "flask"}, set(framework_names))

    def test_get_framework_class(self):
        framework_class = get_framework_class("tornado")
        self.assertIsInstance(framework_class, type)
        self.assertTrue(issubclass(framework_class, Framework))


# test/server/test_helpers.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest

import yaml

from xcube.core.dsio import rimraf
from xcube.server.helpers import ConfigChangeObserver
from xcube.server.server import Server
from .test_server import MockApiContext
from .test_server import MockFramework
from .test_server import mock_extension_registry


class ConfigChangeObserverTest(unittest.TestCase):

    TEST_CONFIG = "config.yaml"

    def tearDown(self) -> None:
        rimraf(self.TEST_CONFIG)

    def test_config_change_observer(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict(create_ctx=MockApiContext)),
        ])

        config = {}
        with open(self.TEST_CONFIG, "w") as fp:
            yaml.safe_dump(config, fp)

        framework = MockFramework()
        server = Server(
            framework, config,
            extension_registry=extension_registry
        )

        server_ctx = server.ctx

        observer = ConfigChangeObserver(server,
                                        [self.TEST_CONFIG],
                                        0.1)

        self.assertEqual(0, framework.call_later_count)
        self.assertIs(server_ctx, server.ctx)

        observer.check()
        self.assertEqual(1, framework.call_later_count)
        self.assertIs(server_ctx, server.ctx)

        observer.check()
        self.assertEqual(2, framework.call_later_count)
        self.assertIs(server_ctx, server.ctx)

        with open(self.TEST_CONFIG, "w") as fp:
            yaml.safe_dump(config, fp)

        observer.check()
        self.assertEqual(3, framework.call_later_count)
        self.assertIsNot(server_ctx, server.ctx)


# test/server/test_server.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest
from typing import Optional

from tornado import concurrent

from xcube.server.api import Api
from xcube.server.api import ApiContext
from xcube.server.api import Context
from xcube.server.server import Server
from xcube.server.server import ServerContext
from xcube.util.jsonschema import JsonArraySchema
from xcube.util.jsonschema import JsonObjectSchema
from xcube.util.jsonschema import JsonStringSchema
from .mocks import MockApiContext
from .mocks import MockFramework
from .mocks import mock_extension_registry
from .mocks import mock_server


class ServerTest(unittest.TestCase):
    def test_basic_props(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict(create_ctx=MockApiContext)),
        ])
        framework = MockFramework()
        server = Server(
            framework, {},
            extension_registry=extension_registry
        )
        self.assertIs(framework, server.framework)
        self.assertIsInstance(server.apis, tuple)
        self.assertIs(server.apis, server.apis)
        self.assertIsInstance(server.open_api_doc, dict)
        self.assertIsInstance(server.ctx, ServerContext)
        self.assertIs(server.ctx, server.ctx)

    def test_accepts_unknown_config_settings(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict(create_ctx=MockApiContext)),
        ])
        framework = MockFramework()
        server = Server(
            framework,
            {
                "i_am_an_unknown_setting": 137
            },
            extension_registry=extension_registry
        )
        self.assertNotIn("i_am_an_unknown_setting",
                         server.ctx.config)

    def test_framework_delegation(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict(create_ctx=MockApiContext)),
        ])
        framework = MockFramework()
        server = Server(
            framework, {},
            extension_registry=extension_registry
        )
        self.assertEqual(1, framework.add_routes_count)
        self.assertEqual(1, framework.update_count)
        self.assertEqual(0, framework.start_count)
        self.assertEqual(0, framework.stop_count)
        server.start()
        self.assertEqual(1, framework.start_count)
        self.assertEqual(0, framework.stop_count)
        server.stop()
        self.assertEqual(1, framework.start_count)
        self.assertEqual(1, framework.stop_count)

    def test_server_ctx(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict(create_ctx=MockApiContext)),
        ])
        server = Server(
            MockFramework(), {},
            extension_registry=extension_registry
        )
        self.assertIsInstance(server.ctx, ServerContext)
        datasets_api_ctx = server.ctx.get_api_ctx('datasets')
        self.assertIsInstance(datasets_api_ctx, MockApiContext)
        self.assertTrue(datasets_api_ctx.on_update_count)
        time_series_api_ctx = server.ctx.get_api_ctx('timeseries')
        self.assertIsNone(time_series_api_ctx)
        self.assertEqual({'address': '0.0.0.0',
                          'port': 8080},
                         server.ctx.config)

    def test_config_schema_effectively_merged(self):
        extension_registry = mock_extension_registry([
            (
                "datasets",
                dict(
                    config_schema=JsonObjectSchema(
                        properties=dict(
                            data_stores=JsonArraySchema(
                                items=JsonObjectSchema(
                                    additional_properties=True
                                )
                            )
                        ),
                        required=['data_stores'],
                        additional_properties=False
                    ))
            ),
        ])
        server = Server(
            MockFramework(),
            {
                "data_stores": []
            },
            extension_registry=extension_registry
        )
        self.assertIsInstance(server.config_schema, JsonObjectSchema)
        self.assertEqual(
            {
                'type': 'object',
                'properties': {
                    'address': {
                        'type': 'string',
                        'default': '0.0.0.0'
                    },
                    'port': {
                        'type': 'integer',
                        'default': 8080
                    },
                    'base_dir': {'type': 'string'},
                    'prefix': {'type': 'string'},
                    'trace_perf': {'type': 'boolean'},
                    'data_stores': {
                        'type': 'array',
                        'items': {
                            'type': 'object',
                            'additionalProperties': True,
                        }
                    },
                },
                'required': ['data_stores'],
                'additionalProperties': True,
            },
            server.config_schema.to_dict()
        )

    def test_config_schema_must_be_object(self):
        extension_registry = mock_extension_registry([
            (
                "datasets",
                dict(
                    config_schema=JsonObjectSchema(
                        properties=dict(address=JsonStringSchema())
                    )
                )
            ),
        ])
        with self.assertRaises(ValueError) as cm:
            Server(
                MockFramework(), {},
                extension_registry=extension_registry
            )
        self.assertEqual(f"API 'datasets':"
                         f" configuration parameter 'address'"
                         f" is already defined.",
                         f'{cm.exception}')

    def test_update_is_effective(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict(create_ctx=MockApiContext)),
            ("timeseries", dict(create_ctx=MockApiContext,
                                required_apis=["datasets"])),
        ])
        server = Server(
            MockFramework(), {},
            extension_registry=extension_registry
        )
        prev_ctx = server.ctx
        server.update({"port": 9090})
        self.assertEqual({'address': '0.0.0.0',
                          'port': 9090},
                         server.ctx.config)
        self.assertIsNot(prev_ctx, server.ctx)

    def test_update_disposes(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict(create_ctx=MockApiContext)),
        ])
        server = Server(
            MockFramework(), {},
            extension_registry=extension_registry
        )
        api_ctx = server.ctx.get_api_ctx("datasets")
        self.assertIsInstance(api_ctx, MockApiContext)
        self.assertEqual(1, api_ctx.on_update_count)
        self.assertEqual(0, api_ctx.on_dispose_count)
        server.update({})
        self.assertEqual(1, api_ctx.on_update_count)
        self.assertEqual(1, api_ctx.on_dispose_count)

    def test_call_later(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict()),
        ])
        framework = MockFramework()
        server = Server(
            framework, {},
            extension_registry=extension_registry
        )
        self.assertEqual(0, framework.call_later_count)
        result = server.call_later(0.01, lambda x: x)
        self.assertIsInstance(result, object)
        self.assertEqual(1, framework.call_later_count)

    def test_run_in_executor(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict()),
        ])
        framework = MockFramework()
        server = Server(
            framework, {},
            extension_registry=extension_registry
        )
        self.assertEqual(0, framework.run_in_executor_count)
        result = server.run_in_executor(None, lambda x: x)
        self.assertIsInstance(result, concurrent.futures.Future)
        self.assertEqual(1, framework.run_in_executor_count)

    def test_apis_loaded_in_order(self):
        extension_registry = mock_extension_registry([
            ("datasets", dict()),
            ("places", dict()),
            ("timeseries", dict()),
            ("stac", dict(required_apis=("datasets", "places"))),
            ("openeo", dict(required_apis=("datasets",
                                           "places", "timeseries"))),
            ("wcs", dict(required_apis=("datasets",))),
            ("wmts", dict(required_apis=("datasets",))),
        ])
        apis = Server.load_apis(extension_registry=extension_registry)
        self.assertIsInstance(apis, tuple)
        self.assertEqual(('datasets',
                          'places',
                          'timeseries',
                          'wcs',
                          'wmts',
                          'stac',
                          'openeo'),
                         tuple(api.name for api in apis))

    def test_illegal_api_context_detected(self):
        # noinspection PyUnusedLocal
        def create_ctx(server_ctx):
            return 42

        extension_registry = mock_extension_registry(
            [('datasets', dict(create_ctx=create_ctx))],
        )

        with self.assertRaises(TypeError) as cm:
            Server(MockFramework(),
                   {},
                   extension_registry=extension_registry)
        self.assertEqual("api_ctx (context of API 'datasets')"
                         " must be an instance of"
                         " <class 'xcube.server.api.ApiContext'>,"
                         " was <class 'int'>",
                         f'{cm.exception}')

    def test_missing_dependency_detected(self):
        extension_registry = mock_extension_registry(
            [('timeseries', dict(required_apis=('datasets',)))]
        )

        with self.assertRaises(ValueError) as cm:
            Server(MockFramework(),
                   {},
                   extension_registry=extension_registry)
        self.assertEqual("API 'timeseries':"
                         " missing API dependency 'datasets'",
                         f'{cm.exception}')


class ServerContextTest(unittest.TestCase):
    def test_basic_props(self):
        server = mock_server()
        config = {}
        server_ctx = ServerContext(server, config)
        self.assertIs(config, server_ctx.config)
        self.assertEqual((), server_ctx.apis)
        self.assertIsInstance(server_ctx.open_api_doc, dict)

    def test_on_update_and_on_dispose(self):
        server = mock_server()
        server_ctx = ServerContext(server, {})
        self.assertIs(None, server_ctx.on_update(None))
        self.assertIs(None, server_ctx.on_dispose())

    def test_async_exec(self):
        framework = MockFramework()
        server = mock_server(framework=framework)
        config = {}
        server_ctx = ServerContext(server, config)

        def my_func(a, b):
            return a + b

        self.assertEqual(0, framework.call_later_count)
        server_ctx.call_later(0.1, my_func, 40, 2)
        self.assertEqual(1, framework.call_later_count)

        self.assertEqual(0, framework.run_in_executor_count)
        server_ctx.run_in_executor(None, my_func, 40, 2)
        self.assertEqual(1, framework.run_in_executor_count)

    class DatasetsContext(ApiContext):

        def on_update(self, prev_ctx: Optional[Context]):
            pass

    class TimeSeriesContext(ApiContext):
        def __init__(self, server_ctx: Context):
            super().__init__(server_ctx)
            self.dataset_ctx = server_ctx.get_api_ctx("datasets")

        def on_update(self, prev_ctx: Optional[Context]):
            pass

    def test_on_update(self):
        api1 = Api("datasets",
                   create_ctx=self.DatasetsContext)
        api2 = Api("timeseries",
                   create_ctx=self.TimeSeriesContext,
                   required_apis=["datasets"])
        config = {}
        server_ctx = ServerContext(mock_server(api_specs=[api1, api2]),
                                   config)
        server_ctx.on_update(None)
        api1_ctx = server_ctx.get_api_ctx('datasets')
        api2_ctx = server_ctx.get_api_ctx('timeseries')

        self.assertIsInstance(api1_ctx, self.DatasetsContext)
        self.assertIsInstance(api2_ctx, self.TimeSeriesContext)

        self.assertIs(server_ctx, api1_ctx.server_ctx)
        self.assertIs(server_ctx, api2_ctx.server_ctx)

        self.assertIs(config, api1_ctx.config)
        self.assertIs(config, api2_ctx.config)

        api21_ctx = api2_ctx.get_api_ctx("datasets")
        self.assertIsInstance(api21_ctx, self.DatasetsContext)


# test/server/test_testing.py

from xcube.server.testing import ServerTestCase


class ServerTestCaseTest(ServerTestCase):

    def test_demonstrate_usage(self):
        url = f'http://localhost:{self.port}/I_do_not_exist'
        response = self.http.request('GET', url)
        self.assertEqual(404, response.status)


# test/util/test_assert.py

from unittest import TestCase

from xcube.util.assertions import assert_condition
from xcube.util.assertions import assert_false
from xcube.util.assertions import assert_given
from xcube.util.assertions import assert_in
from xcube.util.assertions import assert_instance
from xcube.util.assertions import assert_not_none
from xcube.util.assertions import assert_subclass
from xcube.util.assertions import assert_true


class AssertTest(TestCase):
    def test_assert_not_none(self):
        assert_not_none(10, 'x')
        with self.assertRaises(ValueError) as e:
            assert_not_none(None, 'x')
        self.assertEqual(('x must not be None',), e.exception.args)

    def test_assert_given(self):
        assert_given('data.txt', 'x')
        with self.assertRaises(ValueError) as e:
            assert_given('', 'x')
        self.assertEqual(('x must be given',), e.exception.args)

    def test_assert_instance(self):
        assert_instance(10, int, 'x')
        assert_instance('data.txt', (int, str), 'x')
        with self.assertRaises(TypeError) as e:
            assert_instance(0.5, (int, str), 'x')
        self.assertEqual(("x must be an instance of "
                          "(<class 'int'>, <class 'str'>), "
                          "was <class 'float'>",),
                         e.exception.args)

    def test_assert_subclass(self):
        class A:
            pass

        class B(A):
            pass

        assert_subclass(A, A, 'x')
        assert_subclass(B, A, 'x')
        assert_subclass(B, (A, B), 'x')
        with self.assertRaises(TypeError):
            assert_subclass(4, A, 'x')
        with self.assertRaises(TypeError) as e:
            assert_subclass(A, B, 'x')
        self.assertEqual(
            ("x must be a subclass of"
             " <class 'test.util.test_assert.AssertTest"
             ".test_assert_subclass.<locals>.B'>,"
             " was"
             " <class 'test.util.test_assert.AssertTest"
             ".test_assert_subclass.<locals>.A'>",),
            e.exception.args)

    def test_assert_in(self):
        assert_in(2, (1, 2, 3), 'x')
        with self.assertRaises(ValueError) as e:
            assert_in(4, (1, 2, 3), 'x')
        self.assertEqual(('x must be one of (1, 2, 3)',), e.exception.args)

    def test_assert_true(self):
        assert_true(True, 'Should be true')
        with self.assertRaises(ValueError) as e:
            assert_true(False, 'Should be true')
        self.assertEqual(('Should be true',), e.exception.args)

    def test_assert_false(self):
        assert_false(False, 'Should be false')
        with self.assertRaises(ValueError) as e:
            assert_false(True, 'Should be false')
        self.assertEqual(('Should be false',), e.exception.args)

    def test_assert_condition(self):
        assert_condition(True, 'Should be true')
        with self.assertRaises(ValueError) as e:
            assert_condition(False, 'Should be true')
        self.assertEqual(('Should be true',), e.exception.args)


# test/util/test_cache.py

import os
import shutil
import unittest

from xcube.util.cache import CacheStore, Cache, MemoryCacheStore, FileCacheStore, parse_mem_size


class MemoryCacheStoreTest(unittest.TestCase):
    def setUp(self):
        self.cache_store = MemoryCacheStore()
        stored_value_a, size_a = self.cache_store.store_value('a', 42)
        stored_value_b, size_b = self.cache_store.store_value('b', True)
        stored_value_c, size_c = self.cache_store.store_value('c', "S" * 256)
        stored_value_d, size_d = self.cache_store.store_value('d', 3.14)
        self.stored_value_a = stored_value_a
        self.stored_value_b = stored_value_b
        self.stored_value_c = stored_value_c
        self.stored_value_d = stored_value_d
        self.assertEqual(size_a, 28)
        self.assertEqual(size_b, 28)
        self.assertEqual(size_c, 305)
        self.assertEqual(size_d, 24)

    def test_store_value(self):
        self.assertEqual(self.stored_value_a, ['a', 42])
        self.assertEqual(self.stored_value_b, ['b', True])
        self.assertEqual(self.stored_value_c, ['c', "S" * 256])
        self.assertEqual(self.stored_value_d, ['d', 3.14])

    def test_restore_value(self):
        self.assertEqual(self.cache_store.restore_value('a', self.stored_value_a), 42)
        self.assertEqual(self.cache_store.restore_value('b', self.stored_value_b), True)
        with self.assertRaises(ValueError):
            self.cache_store.restore_value('e', self.stored_value_b)

    def test_discard_value(self):
        self.cache_store.discard_value('a', self.stored_value_a)
        self.cache_store.discard_value('b', self.stored_value_b)
        self.assertEqual(self.cache_store.restore_value('a', self.stored_value_a), None)
        self.assertEqual(self.cache_store.restore_value('b', self.stored_value_b), None)
        with self.assertRaises(ValueError):
            self.cache_store.discard_value('e', self.stored_value_b)


class FileCacheStoreTest(unittest.TestCase):
    DIR = '__test_file_cache__'

    def setUp(self):
        if os.path.exists(FileCacheStoreTest.DIR):
            shutil.rmtree(FileCacheStoreTest.DIR, ignore_errors=True)

        os.mkdir(FileCacheStoreTest.DIR)
        try:
            self.cache_store = FileCacheStore(FileCacheStoreTest.DIR, ".dat")
            stored_value_a, size_a = self.cache_store.store_value('a', bytes('abc', 'utf8'))
            stored_value_b, size_b = self.cache_store.store_value('b', bytes('def', 'utf8'))
            stored_value_c, size_c = self.cache_store.store_value('c', bytes('ghi', 'utf8'))
            self.stored_value_a = stored_value_a
            self.stored_value_b = stored_value_b
            self.stored_value_c = stored_value_c
            self.assertEqual(size_a, 3)
            self.assertEqual(size_b, 3)
            self.assertEqual(size_c, 3)
        except Exception as e:
            shutil.rmtree(FileCacheStoreTest.DIR, ignore_errors=True)
            raise e

    def tearDown(self):
        shutil.rmtree(FileCacheStoreTest.DIR, ignore_errors=True)

    def test_store_value(self):
        self.assertEqual(self.stored_value_a, os.path.join(FileCacheStoreTest.DIR, 'a.dat'))
        self.assertEqual(self.stored_value_b, os.path.join(FileCacheStoreTest.DIR, 'b.dat'))
        self.assertEqual(self.stored_value_c, os.path.join(FileCacheStoreTest.DIR, 'c.dat'))

    def test_restore_value(self):
        self.assertEqual(self.cache_store.restore_value('a', self.stored_value_a), bytes('abc', 'utf8'))
        self.assertEqual(self.cache_store.restore_value('b', self.stored_value_b), bytes('def', 'utf8'))
        self.assertEqual(self.cache_store.restore_value('c', self.stored_value_c), bytes('ghi', 'utf8'))
        with self.assertRaises(FileNotFoundError):
            self.cache_store.restore_value('e', self.stored_value_b)

    def test_discard_value(self):
        self.cache_store.discard_value('a', self.stored_value_a)
        self.cache_store.discard_value('b', self.stored_value_b)
        self.cache_store.discard_value('c', self.stored_value_c)
        self.cache_store.discard_value('e', self.stored_value_b)
        with self.assertRaises(FileNotFoundError):
            self.cache_store.restore_value('a', self.stored_value_a)
        with self.assertRaises(FileNotFoundError):
            self.cache_store.restore_value('b', self.stored_value_b)
        with self.assertRaises(FileNotFoundError):
            self.cache_store.restore_value('c', self.stored_value_c)


class TracingCacheStore(CacheStore):
    def __init__(self):
        self.trace = ''

    def can_load_from_key(self, key) -> bool:
        self.trace += 'can_load_from_key(%s);' % key
        return key == 'k5'

    def load_from_key(self, key):
        self.trace += 'load_from_key(%s);' % key
        if key == 'k5':
            return 'S/yyyy', 600
        raise ValueError()

    def store_value(self, key, value):
        self.trace += 'store(%s, %s);' % (key, value)
        return 'S/' + value, 100 * len(value)

    def restore_value(self, key, stored_value):
        self.trace += 'restore(%s, %s);' % (key, stored_value)
        return stored_value[2:]

    def discard_value(self, key, stored_value):
        self.trace += 'discard(%s, %s);' % (key, stored_value)


class CacheTest(unittest.TestCase):
    def setUp(self):
        pass

    def test_store_and_restore_and_discard(self):
        cache_store = TracingCacheStore()
        cache = Cache(store=cache_store, capacity=1000)

        self.assertIs(cache.store, cache_store)
        self.assertEqual(cache.size, 0)
        self.assertEqual(cache.max_size, 750)

        cache_store.trace = ''
        cache.put_value('k1', 'x')
        self.assertEqual(cache.get_value('k1'), 'x')
        self.assertEqual(cache.size, 100)
        self.assertEqual(cache_store.trace, 'store(k1, x);restore(k1, S/x);')

        cache_store.trace = ''
        cache.remove_value('k1')
        self.assertEqual(cache.size, 0)
        self.assertEqual(cache_store.trace, 'discard(k1, S/x);')
        cache_store.trace = ''

        cache_store.trace = ''
        cache.put_value('k1', 'x')
        cache.put_value('k1', 'xx')
        self.assertEqual(cache.get_value('k1'), 'xx')
        self.assertEqual(cache.size, 200)
        self.assertEqual(cache_store.trace, 'store(k1, x);discard(k1, S/x);store(k1, xx);restore(k1, S/xx);')

        cache_store.trace = ''
        cache.remove_value('k1')
        self.assertEqual(cache.size, 0)
        self.assertEqual(cache_store.trace, 'discard(k1, S/xx);')

        cache_store.trace = ''
        cache.put_value('k1', 'x')
        cache.put_value('k2', 'xxx')
        cache.put_value('k3', 'xx')
        self.assertEqual(cache.get_value('k1'), 'x')
        self.assertEqual(cache.get_value('k2'), 'xxx')
        self.assertEqual(cache.get_value('k3'), 'xx')
        self.assertEqual(cache.size, 600)
        self.assertEqual(cache_store.trace, 'store(k1, x);store(k2, xxx);store(k3, xx);'
                                            'restore(k1, S/x);restore(k2, S/xxx);restore(k3, S/xx);')

        cache_store.trace = ''
        cache.put_value('k4', 'xxxx')
        self.assertEqual(cache.size, 600)
        self.assertEqual(cache_store.trace, 'store(k4, xxxx);discard(k1, S/x);discard(k2, S/xxx);')

        cache_store.trace = ''
        cache.clear()
        self.assertEqual(cache.size, 0)

    def test_load_from_key(self):
        cache_store = TracingCacheStore()
        cache = Cache(store=cache_store, capacity=1000)

        cache_store.trace = ''
        self.assertEqual(cache.get_value('k1'), None)
        self.assertEqual(cache.size, 0)
        self.assertEqual(cache_store.trace, 'can_load_from_key(k1);')

        cache_store.trace = ''
        self.assertEqual(cache.get_value('k5'), 'yyyy')
        self.assertEqual(cache.size, 600)
        self.assertEqual(cache_store.trace, 'can_load_from_key(k5);load_from_key(k5);restore(k5, S/yyyy);')


class ParseMemSizeTest(unittest.TestCase):
    def test_parse_mem_size(self):
        self.assertEqual(None, parse_mem_size(""))
        self.assertEqual(None, parse_mem_size("0"))
        self.assertEqual(None, parse_mem_size("0M"))
        self.assertEqual(None, parse_mem_size("off"))
        self.assertEqual(None, parse_mem_size("OFF"))
        self.assertEqual(None, parse_mem_size("None"))
        self.assertEqual(None, parse_mem_size("null"))
        self.assertEqual(None, parse_mem_size("False"))
        self.assertEqual(200001, parse_mem_size("200001"))
        self.assertEqual(200001, parse_mem_size("200001B"))
        self.assertEqual(300000, parse_mem_size("300K"))
        self.assertEqual(3000000, parse_mem_size("3M"))
        self.assertEqual(7000000, parse_mem_size("7m"))
        self.assertEqual(2000000000, parse_mem_size("2g"))
        self.assertEqual(2000000000, parse_mem_size("2G"))
        self.assertEqual(1000000000000, parse_mem_size("1T"))

        with self.assertRaises(ValueError) as cm:
            parse_mem_size("7n")
        self.assertEqual("invalid memory size: '7N'", f"{cm.exception}")

        with self.assertRaises(ValueError) as cm:
            parse_mem_size("-2g")
        self.assertEqual("negative memory size: '-2G'", f"{cm.exception}")


# test/util/test_caseless.py

import unittest

from xcube.util.caseless import caseless_dict


class CaselessDictTest(unittest.TestCase):
    def test_contains(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        self.assertIn('time', d)
        self.assertIn('TIME', d)
        self.assertIn('lon', d)
        self.assertIn('LON', d)
        self.assertNotIn('lat', d)
        self.assertNotIn('LAT', d)

    def test_getitem(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        self.assertEqual('2010-04-09', d['time'])
        self.assertEqual('2010-04-09', d['TIME'])
        self.assertEqual(53.6, d['lon'])
        self.assertEqual(53.6, d['LON'])

        with self.assertRaises(KeyError):
            # noinspection PyUnusedLocal
            var = d['lat']

    def test_setitem(self):
        d = caseless_dict()
        d['TIME'] = '2010-04-09'
        d['LON'] = 53.6
        self.assertIn('time', d)
        self.assertIn('TIME', d)
        self.assertEqual('2010-04-09', d['time'])
        self.assertIn('lon', d)
        self.assertIn('LON', d)
        self.assertEqual(53.6, d['lon'])
        self.assertNotIn('lat', d)
        self.assertNotIn('LAT', d)

    def test_delitem(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        del d['TIME']
        self.assertNotIn('time', d)
        self.assertNotIn('TIME', d)
        self.assertIn('lon', d)
        self.assertIn('LON', d)
        del d['lon']
        self.assertNotIn('lon', d)
        self.assertNotIn('LON', d)

    def test_get(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        self.assertEqual('2010-04-09', d.get('time'))
        self.assertEqual('2010-04-09', d.get('TIME'))
        self.assertEqual(53.6, d.get('lon'))
        self.assertEqual(53.6, d.get('LON'))
        self.assertEqual(None, d.get('lat'))
        self.assertEqual(-3.1, d.get('LAT', -3.1))

    def test_pop(self):
        d = caseless_dict({'TIME': '2010-04-09', 'LON': 53.6})
        v = d.pop('TIME')
        self.assertEqual('2010-04-09', v)
        self.assertNotIn('time', d)
        self.assertNotIn('TIME', d)
        self.assertIn('lon', d)
        self.assertIn('LON', d)
        v = d.pop('lon')
        self.assertEqual(53.6, v)
        self.assertNotIn('lon', d)
        self.assertNotIn('LON', d)


# test/util/test_cmaps.py

import os
from unittest import TestCase

import matplotlib.cm as cm
from matplotlib.colors import LinearSegmentedColormap, ListedColormap, Colormap

from xcube.util.cmaps import get_cmaps, ensure_cmaps_loaded, _get_custom_colormap, get_cmap


class CmapsTest(TestCase):

    def test_get_cmap(self):
        ensure_cmaps_loaded()

        cmap_name, cmap = get_cmap('plasma')
        self.assertEqual('plasma', cmap_name)
        self.assertIsInstance(cmap, Colormap)

        cmap_name, cmap = get_cmap('PLASMA')
        self.assertEqual('viridis', cmap_name)
        self.assertIsInstance(cmap, Colormap)

        cmap_name, cmap = get_cmap('PLASMA', default_cmap_name='magma')
        self.assertEqual('magma', cmap_name)
        self.assertIsInstance(cmap, Colormap)

        with self.assertRaises(ValueError):
            get_cmap('PLASMA', default_cmap_name='MAGMA')

    def test_get_cmaps_returns_singleton(self):
        cmaps = get_cmaps()
        self.assertIs(cmaps, get_cmaps())
        self.assertIs(cmaps, get_cmaps())

    def test_get_cmaps_registers_ocean_colour(self):
        ensure_cmaps_loaded()
        cmap = cm.get_cmap('deep', 256)
        self.assertTrue((type(cmap) is LinearSegmentedColormap) or (type(cmap) is ListedColormap))

    def test_get_cmaps_retruns_equal_size_recs(self):
        cmaps = get_cmaps()
        rec_len = len(cmaps[0])
        self.assertEqual(rec_len, 3)
        for cmap in cmaps:
            self.assertEqual(len(cmap), rec_len)

    def test_get_cmaps_categories(self):
        cmaps = get_cmaps()
        self.assertGreaterEqual(len(cmaps), 8)
        self.assertEqual(cmaps[0][0], 'Perceptually Uniform Sequential')
        self.assertEqual(cmaps[1][0], 'Sequential 1')
        self.assertEqual(cmaps[2][0], 'Sequential 2')
        self.assertEqual(cmaps[3][0], 'Diverging')
        self.assertEqual(cmaps[4][0], 'Qualitative')
        self.assertEqual(cmaps[5][0], 'Ocean')
        self.assertEqual(cmaps[6][0], 'Miscellaneous')
        self.assertEqual(cmaps[7][0], 'Custom SNAP Colormaps')

    def test_get_cmaps_category_descr(self):
        cmaps = get_cmaps()
        self.assertEqual(cmaps[0][1], 'For many applications, a perceptually uniform colormap is the best choice - '
                                      'one in which equal steps in data are perceived as equal steps in the color '
                                      'space')

    def test_get_cmaps_category_tuples(self):
        cmaps = get_cmaps()
        category_tuple = cmaps[0][2]
        self.assertEqual(len(category_tuple), 8)
        self.assertEqual(category_tuple[0][0], 'viridis')
        self.assertEqual(category_tuple[0][1],
                         'iVBORw0KGgoAAAANSUhEUgAAAQAAAAACCAYAAAC3zQLZAAAAzklEQVR4nO2TQZLFIAhEX7dXmyPM/Y8SZwEqMcnU3/9QZTU8GszC6Ee/HQlk5FAsJIENqVGv/piZ3uqf3nX6Vtd+l8D8UwNOLhZL3+BLh796OXvMdWaqtrrqnZ/tjvuZT/0XxnN/5f25z9X7tIMTKzV7/5yrME3NHoPlUzvplgOevOcz6ZO5eCqzOmark1nHDQveHuuYaazZkTcdmE110HJu6doR3tgfPHyL51zNc0fd2xjf0vPukUPL36YBTcpcWArFyY0RTca88cYbXxt/gUOJC8yRF1kAAAAASUVORK5CYII=')

        self.assertEqual(category_tuple[1][0], 'viridis_alpha')
        self.assertEqual(category_tuple[2][0], 'inferno')
        self.assertEqual(category_tuple[3][0], 'inferno_alpha')

    def test_cmocean_category(self):
        cmaps = get_cmaps()
        category_tuple = cmaps[5][2]
        self.assertTrue(len(category_tuple) >= 34)
        self.assertEqual(category_tuple[0][0], 'thermal')
        self.assertEqual(category_tuple[0][1],
                         'iVBORw0KGgoAAAANSUhEUgAAAQAAAAACCAYAAAC3zQLZAAAA2klEQVR4nO2S6xHDMAiDP+FROkL3Xy30RwBju52g8V0OIcnyKxqvtwsD5SfAUPZNE6M4VR2hJTdQeBX6UhlY8xgDY8V24A15pMuIXcQHJo4qwOQYIHlojpT6zWnzqDxRo+/+zFZbR7H2Tx3WvMPf1qDvq+17zz/m7TV97YxHbefEW27ve+7Oe9xZZu3cdXCdr17XokurfvYOcmTXxHJkE2P32ei8eVxww1WJecRlBxZr/cndj+T5MKULbzqm5pnY56MFjnkmPH7cb7xXzvR49RRO3njGM57xt+MDC391Pt11tkYAAAAASUVORK5CYII=')

        self.assertEqual(category_tuple[1][0], 'thermal_alpha')
        self.assertEqual(category_tuple[2][0], 'haline')
        self.assertEqual(category_tuple[3][0], 'haline_alpha')

    def test_get_cmaps_registers_snap_color(self):
        ensure_cmaps_loaded()
        cmap_name = os.path.join(os.path.dirname(__file__), 'chl_DeM2_200.cpd')
        cmap = _get_custom_colormap(cmap_name)
        cm.register_cmap(cmap=cmap)
        self.assertTrue((type(cmap) is LinearSegmentedColormap) or (type(cmap) is ListedColormap))

    def test_get_cmaps_registers_ivalid_snap_color(self):
        ensure_cmaps_loaded()
        cmap_name = os.path.join(os.path.dirname(__file__), 'chl_DeM2_200_invalid_for_testing.cpd')
        with self.assertRaises(ValueError):
            cmap = _get_custom_colormap(cmap_name)
            cm.register_cmap(cmap=cmap)

    def test_get_cmaps_registers_nonexisting_snap_color(self):
        ensure_cmaps_loaded()
        cmap_name = 'test/webapi/im/chl_DeM2_200_not_existing.cpd'
        with self.assertRaises(ValueError):
            cmap = _get_custom_colormap(cmap_name)
            cm.register_cmap(cmap=cmap)


def main():
    cmaps = get_cmaps()

    html_head = '<!DOCTYPE html>\n' + \
                '<html lang="en">\n' + \
                '<head>' + \
                '<meta charset="UTF-8">' + \
                '<title>matplotlib Color Maps</title>' + \
                '</head>\n' + \
                '<body style="padding: 0.2em">\n'

    html_body = ''

    html_foot = '</body>\n' + \
                '</html>\n'

    for cmap_cat, cmap_desc, cmap_bars in cmaps:
        html_body += '    <h2>%s</h2>\n' % cmap_cat
        html_body += '    <p><i>%s</i></p>\n' % cmap_desc
        html_body += '    <table style=border: 0">\n'
        for cmap_bar in cmap_bars:
            cmap_name, cmap_data = cmap_bar
            cmap_image = '<img src="data:image/png;base64,%s" width="100%%" height="20px"/>' % cmap_data

            html_body += '        <tr><td style="width: 5em">%s:</td><td style="width: 40em">%s</td></tr>\n' % (
                cmap_name, cmap_image)
        html_body += '    </table>\n'

    html_page = html_head + html_body + html_foot

    html_filename = 'test_cmaps.html'
    with open(html_filename, 'w') as fp:
        fp.write(html_page)

    import webbrowser
    webbrowser.open_new_tab(html_filename)


if __name__ == '__main__':
    main()


# test/util/test_config.py

import unittest
from io import StringIO

import yaml
from xcube.util.config import flatten_dict, to_name_dict_pair, to_name_dict_pairs, to_resolved_name_dict_pairs, \
    merge_config


# from yaml.parser import ParserError # needs to be kept for last test, which is still not working properly


class ToResolvedNameDictPairsTest(unittest.TestCase):
    def test_to_resolved_name_dict_pairs_and_reject(self):
        container = ['a', 'a1', 'a2', 'b1', 'b2', 'x_c', 'y_c', 'd', 'e', 'e1', 'e_1', 'e_12', 'e_AB']
        resolved = to_resolved_name_dict_pairs([('a*', None),
                                                ('b', {'name': 'B'}),
                                                ('*_c', {'marker': True, 'name': 'C'}),
                                                ('d', {'name': 'D'}),
                                                ('e_??', {'marker': True})],
                                               container)
        self.assertEqual([('a', None),
                          ('a1', None),
                          ('a2', None),
                          # ('b', {'name': 'B'}),  # 'b' is rejected!
                          ('x_c', {'marker': True, 'name': 'C'}),
                          ('y_c', {'marker': True, 'name': 'C'}),
                          ('d', {'name': 'D'}),
                          ('e_12', {'marker': True}),
                          ('e_AB', {'marker': True})],
                         resolved)

    def test_to_resolved_name_dict_pairs_and_keep(self):
        container = ['a', 'a1', 'a2', 'b1', 'b2', 'x_c', 'y_c', 'd', 'e', 'e1', 'e_1', 'e_12', 'e_AB']
        resolved = to_resolved_name_dict_pairs([('a*', None),
                                                ('b', {'name': 'B'}),
                                                ('*_c', {'marker': True, 'name': 'C'}),
                                                ('d', {'name': 'D'}),
                                                ('e_??', {'marker': True})],
                                               container,
                                               keep=True)
        self.assertEqual([('a', None),
                          ('a1', None),
                          ('a2', None),
                          ('b', {'name': 'B'}),  # 'b' is kept!
                          ('x_c', {'marker': True, 'name': 'C'}),
                          ('y_c', {'marker': True, 'name': 'C'}),
                          ('d', {'name': 'D'}),
                          ('e_12', {'marker': True}),
                          ('e_AB', {'marker': True})],
                         resolved)


class ToNameDictPairsTest(unittest.TestCase):
    def test_to_name_dict_pairs_from_list(self):
        parent = ['a*',
                  ('b', 'B'),
                  ('*_c', {'marker': True, 'name': 'C'}),
                  {'d': 'D'},
                  {'e_??': {'marker': True}}]
        pairs = to_name_dict_pairs(parent, default_key='name')
        self.assertEqual([('a*', None),
                          ('b', {'name': 'B'}),
                          ('*_c', {'marker': True, 'name': 'C'}),
                          ('d', {'name': 'D'}),
                          ('e_??', {'marker': True})],
                         pairs)

    def test_to_name_dict_pairs_from_dict(self):
        parent = {'a*': None,
                  'b': 'B',
                  '*_c': {'marker': True, 'name': 'C'},
                  'd': 'D',
                  'e_??': {'marker': True}}
        pairs = to_name_dict_pairs(parent, default_key='name')
        self.assertEqual([('a*', None),
                          ('b', {'name': 'B'}),
                          ('*_c', {'marker': True, 'name': 'C'}),
                          ('d', {'name': 'D'}),
                          ('e_??', {'marker': True})],
                         pairs)


class ToNameDictPairTest(unittest.TestCase):
    def test_name_only(self):
        pair = to_name_dict_pair('a', default_key='name')
        self.assertEqual(('a', None), pair)
        pair = to_name_dict_pair('a?', default_key='name')
        self.assertEqual(('a?', None), pair)

        with self.assertRaises(ValueError) as cm:
            to_name_dict_pair(12, default_key='name')
        self.assertEqual('name must be a string', f'{cm.exception}')

    def test_tuple(self):
        pair = to_name_dict_pair(('a', 'udu'), default_key='name')
        self.assertEqual(('a', dict(name='udu')), pair)
        pair = to_name_dict_pair(('a*', 'udu'), default_key='name')
        self.assertEqual(('a*', dict(name='udu')), pair)
        pair = to_name_dict_pair(('a*', {'name': 'udu'}))
        self.assertEqual(('a*', dict(name='udu')), pair)
        pair = to_name_dict_pair(('a*', {'marker': True, 'name': 'udu'}))
        self.assertEqual(('a*', dict(marker=True, name='udu')), pair)

        with self.assertRaises(ValueError) as cm:
            to_name_dict_pair(('a', 5))
        self.assertEqual("value of 'a' must be a dictionary", f'{cm.exception}')

    def test_dict(self):
        pair = to_name_dict_pair('a', parent=dict(a='udu'), default_key='name')
        self.assertEqual(('a', dict(name='udu')), pair)
        pair = to_name_dict_pair('a*', parent={'a*': 'udu'}, default_key='name')
        self.assertEqual(('a*', dict(name='udu')), pair)

    def test_mapping(self):
        pair = to_name_dict_pair({'a': 'udu'}, default_key='name')
        self.assertEqual(('a', dict(name='udu')), pair)
        pair = to_name_dict_pair({'a*': dict(name='udu')})
        self.assertEqual(('a*', dict(name='udu')), pair)


class FlattenDictTest(unittest.TestCase):

    def test_flatten_dict(self):
        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            flatten_dict(None)

        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            flatten_dict(673)

        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            flatten_dict("?")

        with self.assertRaises(ValueError):
            # noinspection PyTypeChecker
            flatten_dict({0: ""})

        self.assertEqual({}, flatten_dict({}))

        self.assertEqual({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                          "source": "Sentinel-3 OLCI L2 surface observation"
                          },
                         flatten_dict({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                                       "source": "Sentinel-3 OLCI L2 surface observation"
                                       }))

        self.assertEqual({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                          "date_created": "2018-05-30",
                          "date_modified": "2018-05-30",
                          "date_issued": "2018-06-01"
                          },
                         flatten_dict({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                                       "date": {
                                           "created": "2018-05-30",
                                           "modified": "2018-05-30",
                                           "issued": "2018-06-01"
                                       }}))

        self.assertEqual({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                          "creator_name": "BC",
                          "creator_url": "http://www.bc.de"
                          },
                         flatten_dict({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                                       "creator": [
                                           {"name": "BC",
                                            "url": "http://www.bc.de"
                                            }
                                       ]}))

        self.assertEqual({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                          "creator_name": "BC, ACME",
                          "creator_url": "http://www.bc.de, http://acme.com"
                          },
                         flatten_dict({"title": "DCS4COP Sentinel-3 OLCI L2C Data Cube",
                                       "creator": [
                                           {"name": "BC",
                                            "url": "http://www.bc.de"
                                            },
                                           {"name": "ACME",
                                            "url": "http://acme.com"
                                            }
                                       ]}))

    def test_from_yaml(self):
        stream = StringIO(TEST_JSON)
        d = yaml.full_load(stream)
        d = flatten_dict(d['output_metadata'])
        self.assertEqual(17, len(d))
        self.assertEqual('DCS4COP Sentinel-3 OLCI L2C Data Cube',
                         d.get('title'))
        self.assertEqual('Brockmann Consult GmbH, Royal Belgian Institute for Natural Sciences (RBINS)',
                         d.get('creator_name'))
        self.assertEqual('https://www.brockmann-consult.de, http://odnature.naturalsciences.be/remsem/',
                         d.get('creator_url'))
        self.assertEqual("2018-05-30", d.get('date_created'))
        self.assertEqual("2018-06-01", d.get('date_issued'))
        self.assertEqual(0.0, d.get('geospatial_lon_min'))
        self.assertEqual(5.0, d.get('geospatial_lon_max'))
        self.assertEqual(50.0, d.get('geospatial_lat_min'))
        self.assertEqual(52.5, d.get('geospatial_lat_max'))
        self.assertEqual('degrees_east', d.get('geospatial_lon_units'))
        self.assertEqual('degrees_north', d.get('geospatial_lat_units'))
        self.assertEqual(0.0025, d.get('geospatial_lon_resolution'))
        self.assertEqual(0.0025, d.get('geospatial_lat_resolution'))
        self.assertEqual('2016-10-01', d.get('time_coverage_start'))
        self.assertEqual('2017-10-01T12:00:10', d.get('time_coverage_end'))
        self.assertEqual('P1Y', d.get('time_coverage_duration'))
        self.assertEqual('1D', d.get('time_coverage_resolution'))


TEST_JSON = """
output_metadata:
  # CF: A succinct description of what is in the dataset.
  title: "DCS4COP Sentinel-3 OLCI L2C Data Cube"
  
  # The data creator's name, URL, and email.
  # The "institution" attribute will be used if the "creator_name" attribute does not exist.
  creator:
    - name: "Brockmann Consult GmbH"
      url: "https://www.brockmann-consult.de"
    - name: "Royal Belgian Institute for Natural Sciences (RBINS)"
      url: "http://odnature.naturalsciences.be/remsem/"
  
  date:
    # The date on which the data was created.
    created:  "2018-05-30"
    # The date on which this data was formally issued.
    issued:   "2018-06-01"
  
  geospatial_lon:
    min:  0.0
    max:  5.0
    units: "degrees_east"
    resolution: 0.0025
  
  geospatial_lat:
    min: 50.0
    max: 52.5
    units: "degrees_north"
    resolution: 0.0025
  
  time_coverage:
    start:      2016-10-01
    end:        2017-10-01T12:00:10
    duration:   "P1Y"
    resolution: "1D"
"""


class MergeDictsTest(unittest.TestCase):
    def test_single_dict(self):
        first_dict = {}
        actual_dict = merge_config(first_dict)
        self.assertIs(first_dict, actual_dict)

    def test_two_dicts(self):
        first_dict = {'a': 'name'}
        second_dict = {'b': 'age'}
        actual_dict = merge_config(first_dict, second_dict)
        self.assertEqual({'a': 'name', 'b': 'age'}, actual_dict)

    def test_three_dicts(self):
        first_dict = {'a': 'name'}
        second_dict = {'b': 'age'}
        third_dict = {'c': 'hair'}
        actual_dict = merge_config(first_dict, second_dict, third_dict)
        self.assertEqual({'a': 'name', 'b': 'age', 'c': 'hair'}, actual_dict)

    def test_order_matters(self):
        first_dict = {'a': 'name', 'b': 42}
        sec_dict = {'a': 'age'}
        third_dict = {'b': 15}
        actual_dict = merge_config(first_dict, sec_dict, third_dict)
        self.assertEqual({'a': 'age', 'b': 15}, actual_dict)

    def test_merge_deep(self):
        first_dict = {'a': dict(b=42, c=15), 'o': 105}
        second_dict = {'a': dict(c=25)}
        actual_dict = merge_config(first_dict, second_dict)
        self.assertEqual({'a': dict(b=42, c=25), 'o': 105}, actual_dict)

    def test_merge_dict_value(self):
        first_dict = {'o': 105}
        second_dict = {'a': dict(c=25)}
        actual_dict = merge_config(first_dict, second_dict)
        self.assertEqual({'a': dict(c=25), 'o': 105}, actual_dict)


# test/util/test_dask.py

import unittest

import numpy as np

from xcube.util.dask import _NestedList
from xcube.util.dask import compute_array_from_func
from xcube.util.dask import get_chunk_sizes
from xcube.util.dask import get_chunk_slice_tuples


class DaskTest(unittest.TestCase):
    def test_from_func_8x10_3x4(self):
        self._assert_from_func(shape=(8, 10), chunks=(3, 4))

    def test_from_func_80x100_3x4(self):
        self._assert_from_func(shape=(80, 100), chunks=(3, 4))

    def test_from_func_80x100_7x10(self):
        self._assert_from_func(shape=(80, 100), chunks=(7, 10))

    def test_from_func_80x100_40x25(self):
        self._assert_from_func(shape=(80, 100), chunks=(40, 25))

    def _assert_from_func(self, shape, chunks):
        def my_func(array_shape, array_dtype, block_shape, block_slices):
            bsy, bsx = block_slices
            bh, bw = block_shape
            w = array_shape[-1]
            i0 = bsx[0]
            j0 = bsy[0]
            a = np.ndarray((bh, bw), dtype=array_dtype)
            for j in range(bh):
                for i in range(bw):
                    a[j, i] = 0.1 * (i0 + i + (j0 + j) * w)
            return a

        a = compute_array_from_func(my_func,
                                    shape,
                                    chunks,
                                    np.float64,
                                    ctx_arg_names=['shape', 'dtype', 'block_shape', 'block_slices'])

        self.assertIsNotNone(a)
        self.assertEqual(shape, a.shape)
        self.assertEqual(tuple(get_chunk_sizes(shape, chunks)), a.chunks)
        self.assertEqual(np.float64, a.dtype)

        h, w = shape
        n = w * h

        # Compute result
        actual = np.array(a)
        expected = (0.1 * np.linspace(0, n - 1, n, dtype=np.float64)).reshape(shape)
        np.testing.assert_almost_equal(actual, expected)

    def test_get_chunk_sizes(self):
        chunks = get_chunk_sizes((100, 100, 40), (30, 25, 50))
        self.assertEqual([(30, 30, 30, 10),
                          (25, 25, 25, 25),
                          (40,)],
                         list(chunks))

        chunks = get_chunk_sizes((100, 100, 40), (100, 100, 40))
        self.assertEqual([(100,),
                          (100,),
                          (40,)],
                         list(chunks))

    def test_get_chunk_slices_tuples(self):
        chunk_slices_tuples = get_chunk_slice_tuples([(30, 30, 30, 10),
                                                      (25, 25, 25, 25),
                                                      (40,)])
        self.assertEqual([(slice(0, 30),
                           slice(30, 60),
                           slice(60, 90),
                           slice(90, 100)),
                          (slice(0, 25),
                           slice(25, 50),
                           slice(50, 75),
                           slice(75, 100)),
                          (slice(0, 40),)],
                         list(chunk_slices_tuples))


class NestedListTest(unittest.TestCase):
    def test_len(self):
        nl = _NestedList((4,))
        self.assertEqual(4, len(nl))

        nl = _NestedList((3, 4))
        self.assertEqual(3, len(nl))

        nl = _NestedList((2, 3, 4))
        self.assertEqual(2, len(nl))

    def test_shape(self):
        nl = _NestedList([4])
        self.assertEqual((4,), nl.shape)

        nl = _NestedList([3, 4])
        self.assertEqual((3, 4), nl.shape)

        nl = _NestedList([2, 3, 4])
        self.assertEqual((2, 3, 4), nl.shape)

    def test_data(self):
        nl = _NestedList((4,))
        self.assertEqual([None, None, None, None], nl.data)

        nl = _NestedList((3, 4))
        self.assertEqual([[None, None, None, None],
                          [None, None, None, None],
                          [None, None, None, None]], nl.data)

        nl = _NestedList((2, 3, 4))
        self.assertEqual([[[None, None, None, None],
                           [None, None, None, None],
                           [None, None, None, None]],
                          [[None, None, None, None],
                           [None, None, None, None],
                           [None, None, None, None]]], nl.data)

    def test_setget(self):
        nl = _NestedList((4,), fill_value=0)
        nl[1] = 2
        self.assertEqual([0, 2, 0, 0], nl.data)
        self.assertEqual(0, nl.data[0])
        self.assertEqual(2, nl.data[1])
        self.assertEqual(0, nl.data[2])
        nl[2:4] = [6, 7]
        self.assertEqual([0, 2, 6, 7], nl.data)

        nl = _NestedList((4, 3), fill_value=True)
        nl[2, 1] = False
        self.assertEqual([[True, True, True],
                          [True, True, True],
                          [True, False, True],
                          [True, True, True]], nl.data)
        nl[0, 0:2] = [False, False]
        self.assertEqual([[False, False, True],
                          [True, True, True],
                          [True, False, True],
                          [True, True, True]], nl.data)


# test/util/test_expression.py

import unittest
from collections import namedtuple

import numpy as np
import numpy.testing as npt
import xarray as xr

from xcube.util.expression import transpile_expr
from xcube.util.expression import compute_expr
from xcube.util.expression import compute_array_expr


class ComputeArrayExprTest(unittest.TestCase):
    # noinspection PyMethodMayBeStatic
    def test_valid_exprs(self):
        namespace = dict(a=np.array([0.1, 0.3, 0.1, 0.7, 0.4, 0.9]),
                         b=np.array([0.2, 0.1, 0.3, 0.2, 0.4, 0.8]),
                         np=np,
                         xr=xr)

        value = compute_array_expr('a + 1', namespace=namespace)
        npt.assert_array_almost_equal(value,
                                      np.array([1.1, 1.3, 1.1, 1.7, 1.4, 1.9]))

        value = compute_array_expr('a * b', namespace=namespace)
        npt.assert_array_almost_equal(value,
                                      np.array([0.02, 0.03, 0.03, 0.14, 0.16, 0.72]))

        value = compute_array_expr('max(a, b)', namespace=namespace)
        npt.assert_array_almost_equal(value,
                                      np.array([0.2, 0.3, 0.3, 0.7, 0.4, 0.9]))

        value = compute_array_expr('a > b', namespace=namespace)
        npt.assert_equal(value,
                         np.array([False, True, False, True, False, True]))

        value = compute_array_expr('a == b', namespace=namespace)
        npt.assert_equal(value,
                         np.array([False, False, False, False, True, False]))

        # This weirdo expression is a result of translating SNAP conditional expressions to Python.
        value = compute_array_expr('a > 0.35 if a else b', namespace=namespace)
        npt.assert_equal(value,
                         np.array([0.2, 0.1, 0.3, 0.7, 0.4, 0.9]))

        # We actually mean
        value = compute_array_expr('where(a > 0.35, a, b)', namespace=namespace)
        npt.assert_equal(value,
                         np.array([0.2, 0.1, 0.3, 0.7, 0.4, 0.9]))

    def test_invalid_exprs(self):
        with self.assertRaises(ValueError) as cm:
            compute_expr('20. -')
        self.assertEqual("failed computing expression '20. -': "
                         "unexpected EOF while parsing (<string>, line 1)",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            compute_expr('2 - a')
        self.assertEqual("failed computing expression '2 - a': "
                         "name 'a' is not defined",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            compute_expr('b < 1', result_name="mask of 'CHL'")
        self.assertEqual("failed computing mask of 'CHL' from "
                         "expression 'b < 1': name 'b' is not defined",
                         f'{cm.exception}')

    def test_complex_case(self):
        expr = ('(not quality_flags.invalid'
                ' and not pixel_classif_flags.IDEPIX_CLOUD'
                ' and not pixel_classif_flags.IDEPIX_CLOUD_BUFFER'
                ' and not pixel_classif_flags.IDEPIX_CLOUD_SHADOW'
                ' and not pixel_classif_flags.IDEPIX_SNOW_ICE'
                ' and not (c2rcc_flags.Rtosa_OOS and conc_chl > 1.0)'
                ' and not c2rcc_flags.Rtosa_OOR'
                ' and not c2rcc_flags.Rhow_OOR'
                ' and not (c2rcc_flags.Cloud_risk and immersed_cyanobacteria == 0)'
                ' and floating_vegetation == 0'
                ' and conc_chl > 0.01'
                ' and not (floating_cyanobacteria == 1 or chl_pitarch > 500))')

        quality_flags = namedtuple('quality_flags',
                                   ['invalid'])
        quality_flags.invalid = np.array([0])
        pixel_classif_flags = namedtuple('pixel_classif_flags',
                                         ['IDEPIX_CLOUD',
                                          'IDEPIX_CLOUD_BUFFER',
                                          'IDEPIX_CLOUD_SHADOW',
                                          'IDEPIX_SNOW_ICE'])
        pixel_classif_flags.IDEPIX_CLOUD = np.array([0])
        pixel_classif_flags.IDEPIX_CLOUD_BUFFER = np.array([0])
        pixel_classif_flags.IDEPIX_CLOUD_SHADOW = np.array([0])
        pixel_classif_flags.IDEPIX_SNOW_ICE = np.array([0])
        c2rcc_flags = namedtuple('c2rcc_flags',
                                 ['Rtosa_OOS',
                                  'Rtosa_OOR',
                                  'Rhow_OOR',
                                  'Cloud_risk'])
        c2rcc_flags.Rtosa_OOS = np.array([0])
        c2rcc_flags.Rtosa_OOR = np.array([0])
        c2rcc_flags.Rhow_OOR = np.array([0])
        c2rcc_flags.Cloud_risk = np.array([0])
        namespace = dict(
            np=np,
            quality_flags=quality_flags,
            pixel_classif_flags=pixel_classif_flags,
            c2rcc_flags=c2rcc_flags,
            immersed_cyanobacteria=np.array([0]),
            floating_cyanobacteria=np.array([0]),
            floating_vegetation=np.array([0]),
            conc_chl=np.array([0]),
            chl_pitarch=np.array([0]),
        )

        actual_value = compute_array_expr(expr, namespace=namespace)
        expected_value = 0
        npt.assert_array_almost_equal(actual_value, np.array([expected_value]))

        namespace['conc_chl'] = np.array([0.2])
        actual_value = compute_array_expr(expr, namespace=namespace)
        expected_value = 1
        npt.assert_array_almost_equal(actual_value, np.array([expected_value]))

        pixel_classif_flags.IDEPIX_CLOUD_SHADOW = np.array([0.2])
        actual_value = compute_array_expr(expr, namespace=namespace)
        expected_value = 0
        npt.assert_array_almost_equal(actual_value, np.array([expected_value]))


class ComputeExprTest(unittest.TestCase):
    def test_valid_exprs(self):
        namespace = dict(a=3)
        self.assertEqual(2, compute_expr('2'))
        self.assertEqual("u", compute_expr('"u"'))
        self.assertEqual(3, compute_expr('a', namespace=namespace))
        self.assertEqual(True, compute_expr('True'))
        self.assertEqual(True, compute_expr('a == 3', namespace=namespace))
        self.assertEqual(5, compute_expr('a + 2', namespace=namespace))

    def test_invalid_exprs(self):
        with self.assertRaises(ValueError) as cm:
            compute_expr('20. -')
        self.assertEqual("failed computing expression '20. -': "
                         "unexpected EOF while parsing (<string>, line 1)",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            compute_expr('2 - a')
        self.assertEqual("failed computing expression '2 - a': "
                         "name 'a' is not defined",
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            compute_expr('b < 1', result_name="mask of 'CHL'")
        self.assertEqual("failed computing mask of 'CHL' from "
                         "expression 'b < 1': name 'b' is not defined",
                         f'{cm.exception}')


class TranspileExprTest(unittest.TestCase):
    def test_unary(self):
        self.assertEqual('--+-x',
                         transpile_expr('-(-+(-x))'))
        self.assertEqual('-(a - b - c)',
                         transpile_expr('-((a-b)-c)'))
        self.assertEqual('np.logical_not(x)',
                         transpile_expr('not x'))
        self.assertEqual('np.logical_not(-x)',
                         transpile_expr('not -x'))
        self.assertEqual('np.logical_not(np.logical_not(x))',
                         transpile_expr('not not x'))

    def test_binary(self):
        self.assertEqual('a - b - c - d',
                         transpile_expr('a-b-c-d'))
        self.assertEqual('a - b - c - d',
                         transpile_expr('(a-b)-c-d'))
        self.assertEqual('a - b - c - d',
                         transpile_expr('(a-b-c)-d'))
        self.assertEqual('a - (b - c) - d',
                         transpile_expr('a-(b-c)-d'))
        self.assertEqual('a - (b - c - d)',
                         transpile_expr('a-(b-c-d)'))
        self.assertEqual('a - b - (c - d)',
                         transpile_expr('a-b-(c-d)'))
        self.assertEqual('a - (b - (c - d))',
                         transpile_expr('a-(b-(c-d))'))
        self.assertEqual('a - (b - c) - d',
                         transpile_expr('(a-(b-c))-d'))

        self.assertEqual('a - ---b',
                         transpile_expr('a----b'))
        self.assertEqual('---a - b',
                         transpile_expr('---a-b'))

        self.assertEqual('a * b + c / d',
                         transpile_expr('a*b+c/d'))
        self.assertEqual('a + b * c - d',
                         transpile_expr('a+b*c-d'))
        self.assertEqual('(a + b) * (c - d)',
                         transpile_expr('(a+b)*(c-d)'))

        self.assertEqual('np.power(a, b)',
                         transpile_expr('a ** b'))

    def test_bool(self):
        self.assertEqual('np.logical_and(np.logical_and(a, b), c)',
                         transpile_expr('a and b and c'))
        self.assertEqual('np.logical_and(np.logical_and(a, b), c)',
                         transpile_expr('(a and b) and c'))
        self.assertEqual('np.logical_and(a, np.logical_and(b, c))',
                         transpile_expr('a and (b and c)'))
        self.assertEqual('np.logical_and(np.logical_or(a, b), '
                         'np.logical_not(np.logical_or(c, '
                         'np.logical_not(d))))',
                         transpile_expr('(a or b) and not (c or not d)'))

        self.assertEqual('np.logical_or(np.logical_and(a, b), '
                         'np.logical_and(c, d))',
                         transpile_expr('a and b or c and d'))
        self.assertEqual('np.logical_or(np.logical_or(a, '
                         'np.logical_and(b, c)), d)',
                         transpile_expr('a or b and c or d'))
        self.assertEqual('np.logical_and(np.logical_or(a, b), '
                         'np.logical_or(c, d))',
                         transpile_expr('(a or b) and (c or d)'))

    def test_compare(self):
        self.assertEqual('a < 2',
                         transpile_expr('a < 2'))
        self.assertEqual('a != 2',
                         transpile_expr('a != 2'))
        self.assertEqual('np.isnan(a)',
                         transpile_expr('a == NaN'))
        self.assertEqual('np.logical_not(np.isnan(a))',
                         transpile_expr('a != NaN'))
        self.assertEqual('np.isnan(a)',
                         transpile_expr('NaN == a'))
        self.assertEqual('np.logical_not(np.isnan(a))',
                         transpile_expr('NaN != a'))
        self.assertEqual('a is not Null',
                         transpile_expr('a is not Null'))
        self.assertEqual('a in data',
                         transpile_expr('a in data'))

        with self.assertRaises(ValueError) as cm:
            transpile_expr('a >= 2 == b')
        x8 = str(cm.exception)
        self.assertEqual('expression "a >= 2 == b" uses an n-ary '
                         'comparison, but only binary are supported', x8)

        with self.assertRaises(ValueError) as cm:
            transpile_expr('0 < x <= 1')
        x9 = str(cm.exception)
        self.assertEqual('expression "0 < x <= 1" uses an n-ary '
                         'comparison, but only binary are supported', x9)

    def test_attributes(self):
        self.assertEqual('a.b',
                         transpile_expr('a.b'))

    def test_functions(self):
        self.assertEqual('np.sin(x)',
                         transpile_expr('sin(x)'))
        self.assertEqual('np.fmin(x, y)',
                         transpile_expr('min(x, y)'))
        self.assertEqual('np.fmax(x, y)',
                         transpile_expr('max(x, y)'))
        self.assertEqual('np.isnan(x, y)',
                         transpile_expr('isnan(x, y)'))

    def test_conditional(self):
        # The following conditional expr looks wrong but this 
        # is how it looks like after translating
        # from SNAP expression 'a >= 0.0 ? a : NaN'
        self.assertEqual('xr.where(a >= 0.0, a, NaN)',
                         transpile_expr('a >= 0.0 if a else NaN'))
        self.assertEqual('xr.where(a >= 0.0, a, xr.where(b >= 0.0, b, NaN))',
                         transpile_expr('a >= 0.0 if a else b >= 0.0 if b else NaN'))

    def test_where(self):
        self.assertEqual('xr.where(a >= 0.0, a, NaN)',
                         transpile_expr('where(a >= 0.0, a, NaN)'))
        self.assertEqual('xr.where(a >= 0.0, a, NaN)',
                         transpile_expr('xr.where(a >= 0.0, a, NaN)'))
        self.assertEqual('np.where(a >= 0.0, a, NaN)',
                         transpile_expr('np.where(a >= 0.0, a, NaN)'))
        # xarray.DataArray.where() method:
        self.assertEqual('a.where(a.x >= 0.0)',
                         transpile_expr('a.where(a.x >= 0.0)'))

    def test_mixed(self):
        self.assertEqual('a + np.sin(x + 2.8)',
                         transpile_expr('a+sin(x + 2.8)'))
        self.assertEqual('a + np.fmax(1, np.sin(x + 2.8), np.power(x, 0.5))',
                         transpile_expr('a+max(1, sin(x+2.8), x**0.5)'))

        expr = ('(not quality_flags.invalid'
                ' and not pixel_classif_flags.IDEPIX_CLOUD'
                ' and not pixel_classif_flags.IDEPIX_CLOUD_BUFFER'
                ' and not pixel_classif_flags.IDEPIX_CLOUD_SHADOW'
                ' and not pixel_classif_flags.IDEPIX_SNOW_ICE'
                ' and not (c2rcc_flags.Rtosa_OOS and conc_chl > 1.0)'
                ' and not c2rcc_flags.Rtosa_OOR'
                ' and not c2rcc_flags.Rhow_OOR'
                ' and not (c2rcc_flags.Cloud_risk and immersed_cyanobacteria == 0)'
                ' and floating_vegetation == 0'
                ' and conc_chl > 0.01'
                ' and not (floating_cyanobacteria == 1 or chl_pitarch > 500))')

        self.assertEqual('np.logical_and(np.logical_and('
                         'np.logical_and(np.logical_and('
                         'np.logical_and(np.logical_and('
                         'np.logical_and(np.logical_and('
                         'np.logical_and(np.logical_and('
                         'np.logical_and('
                         'np.logical_not(quality_flags.invalid), '
                         'np.logical_not(pixel_classif_flags.IDEPIX_CLOUD)), '
                         'np.logical_not(pixel_classif_flags.IDEPIX_CLOUD_BUFFER)), '
                         'np.logical_not(pixel_classif_flags.IDEPIX_CLOUD_SHADOW)), '
                         'np.logical_not(pixel_classif_flags.IDEPIX_SNOW_ICE)), '
                         'np.logical_not('
                         'np.logical_and(c2rcc_flags.Rtosa_OOS, conc_chl > 1.0))), '
                         'np.logical_not(c2rcc_flags.Rtosa_OOR)), '
                         'np.logical_not(c2rcc_flags.Rhow_OOR)), '
                         'np.logical_not('
                         'np.logical_and(c2rcc_flags.Cloud_risk, immersed_cyanobacteria == 0))),'
                         ' floating_vegetation == 0), conc_chl > 0.01), '
                         'np.logical_not('
                         'np.logical_or(floating_cyanobacteria == 1, chl_pitarch > 500)))',
                         transpile_expr(expr))


# test/util/test_extend.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest

from xcube.util.extend import extend


class ExtendTest(unittest.TestCase):
    def test_decorator(self):
        class MyBase:
            pass

        @extend(MyBase, "my_ext")
        class MyExt:
            """My pretty extension."""

            def __init__(self, base: MyBase):
                self._base = base

        self.assertTrue(hasattr(MyBase, "my_ext"))
        # noinspection PyUnresolvedReferences
        self.assertEqual("My pretty extension.", MyBase.my_ext.__doc__)

        my_base = MyBase()
        self.assertTrue(hasattr(my_base, "my_ext"))
        # noinspection PyUnresolvedReferences
        my_ext = my_base.my_ext
        self.assertIsInstance(my_ext, MyExt)
        self.assertEqual("My pretty extension.", my_ext.__doc__)
        self.assertIsInstance(my_ext._base, MyBase)

    def test_class_handler(self):
        class MyBase:
            extensions = []

            @classmethod
            def add_extension(cls, ext_item):
                MyBase.extensions.append(ext_item)

        @extend(MyBase, 'my_ext_1', class_handler='add_extension')
        class MyExt1:
            def __init__(self, base: MyBase):
                pass

        @extend(MyBase, 'my_ext_2', class_handler='add_extension')
        class MyExt2:
            def __init__(self, base: MyBase):
                pass

        self.assertEqual([('my_ext_1', MyExt1),
                          ('my_ext_2', MyExt2)],
                         MyBase.extensions)

    def test_instance_handler(self):
        class MyBase:
            def __init__(self):
                self.extensions = []

            def add_extension(self, ext_item):
                self.extensions.append(ext_item)

        @extend(MyBase, 'my_ext_1', inst_handler='add_extension')
        class MyExt1:
            def __init__(self, base: MyBase):
                pass

        @extend(MyBase, 'my_ext_2', inst_handler='add_extension')
        class MyExt2:
            def __init__(self, base: MyBase):
                pass

        my_base = MyBase()
        self.assertEqual([],
                         my_base.extensions)
        self.assertIsInstance(my_base.my_ext_1, MyExt1)
        self.assertIsInstance(my_base.my_ext_2, MyExt2)
        self.assertEqual(2,
                         len(my_base.extensions))

    def test_overwrite_docstring(self):
        class MyBase:
            pass

        class MyExt:
            """My pretty extension."""

            def __init__(self, base: MyBase):
                self._base = base

        extend(MyBase, "my_ext", doc="A better doc.")(MyExt)

        # noinspection PyUnresolvedReferences
        self.assertEqual("A better doc.", MyBase.my_ext.__doc__)

    def test_property_exists(self):
        class MyBase:
            @property
            def my_ext(self):
                return 42

        class MyExt:
            pass

        with self.assertRaises(ValueError) as cm:
            extend(MyBase, "my_ext")(MyExt)
        self.assertEqual('a property named my_ext'
                         ' already exists in class MyBase',
                         f'{cm.exception}')

    def test_illegal_property_name(self):
        class MyBase:
            pass

        class MyExt:
            pass

        with self.assertRaises(ValueError) as cm:
            extend(MyBase, "my ext")(MyExt)
        self.assertEqual("name must be a valid identifier,"
                         " but was 'my ext'",
                         f'{cm.exception}')

    def test_illegal_base_class(self):
        class MyExt:
            pass

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            extend("base_class", "my_ext")(MyExt)
        self.assertEqual('base_class must be a class type,'
                         ' but was str',
                         f'{cm.exception}')

    def test_illegal_ext_class(self):
        class MyBase:
            pass

        with self.assertRaises(ValueError) as cm:
            # noinspection PyTypeChecker
            extend(MyBase, "my_ext")("ext_class")
        self.assertEqual('the extend() decorator'
                         ' can be used with classes only',
                         f'{cm.exception}')


# test/util/test_extension.py

import unittest
from typing import Any

from xcube.util.extension import ExtensionRegistry, Extension, get_extension_registry
from xcube.util.extension import import_component


class A:
    def __init__(self, **kwargs):
        self.kwargs = kwargs


class B(A):
    pass


class ExtensionRegistryTest(unittest.TestCase):
    def test_get_extension_reg(self):
        self.assertIsInstance(get_extension_registry(), ExtensionRegistry)

    def test_get_extension_illegal(self):
        ext_reg = ExtensionRegistry()
        self.assertEqual(False, ext_reg.has_extension('A', 'test'))
        self.assertEqual(None, ext_reg.get_extension('A', 'test'))

    def test_get_component_illegal(self):
        ext_reg = ExtensionRegistry()
        with self.assertRaises(ValueError) as cm:
            ext_reg.get_component('A', 'test')
        self.assertEqual("extension 'test' not found for extension point 'A'", f'{cm.exception}')

    def test_protocol(self):
        ext_reg = ExtensionRegistry()

        a_obj = A()

        def b_loader(ext):
            return B(name=ext.name)

        a_ext = ext_reg.add_extension(component=a_obj, point='A', name='test')
        b_ext = ext_reg.add_extension(loader=b_loader, point='B', name='test')

        self.assertEqual(True, ext_reg.has_extension('A', 'test'))
        self.assertIsInstance(a_ext, Extension)
        self.assertEqual('test', a_ext.name)
        self.assertEqual('A', a_ext.point)
        self.assertEqual(False, a_ext.is_lazy)
        self.assertIs(a_obj, a_ext.component)
        self.assertIs(a_obj, a_ext.component)
        self.assertIs(a_obj, ext_reg.get_component('A', 'test'))
        self.assertIs(a_ext, ext_reg.get_extension('A', 'test'))

        self.assertEqual(True, ext_reg.has_extension('B', 'test'))
        self.assertIsInstance(b_ext, Extension)
        self.assertEqual('test', b_ext.name)
        self.assertEqual('B', b_ext.point)
        self.assertEqual(True, b_ext.is_lazy)
        b_obj = b_ext.component
        self.assertIs(b_obj, b_ext.component)
        self.assertIs(b_obj, ext_reg.get_component('B', 'test'))
        self.assertIs(b_ext, ext_reg.get_extension('B', 'test'))
        self.assertEqual({'name': 'test'}, b_obj.kwargs)

        self.assertEqual([a_ext], ext_reg.find_extensions('A'))
        self.assertEqual([b_ext], ext_reg.find_extensions('B'))
        self.assertEqual([], ext_reg.find_extensions('C'))

        self.assertEqual([a_obj], ext_reg.find_components('A'))
        self.assertEqual([b_obj], ext_reg.find_components('B'))
        self.assertEqual([], ext_reg.find_components('C'))

        ext_reg.remove_extension('A', 'test')
        self.assertEqual(False, ext_reg.has_extension('A', 'test'))

        ext_reg.remove_extension('B', 'test')
        self.assertEqual(False, ext_reg.has_extension('B', 'test'))

    def test_find(self):
        ext_reg = ExtensionRegistry()
        obj1 = A()
        obj2 = A()
        obj3 = A()
        obj4 = B()
        obj5 = B()
        obj6 = B()

        # noinspection PyUnusedLocal
        def load_obj3(ext: Extension):
            return obj3

        # noinspection PyUnusedLocal
        def load_obj4(ext: Extension):
            return obj4

        ext_reg.add_extension(component=obj1, point='A', name='a1', description='knorg')
        ext_reg.add_extension(component=obj2, point='A', name='a2', description='gnatz')
        ext_reg.add_extension(loader=load_obj3, point='A', name='a3', description='gnatz')
        ext_reg.add_extension(loader=load_obj4, point='B', name='b1', description='knorg')
        ext_reg.add_extension(component=obj5, point='B', name='b2', description='gnatz')
        ext_reg.add_extension(component=obj6, point='B', name='b3', description='knorg')

        def is_knorg(ext: Extension):
            return ext.metadata.get('description') == 'knorg'

        def is_gnatz(ext: Extension):
            return ext.metadata.get('description') == 'gnatz'

        result = ext_reg.find_extensions('A', predicate=is_knorg)
        self.assertEqual(1, len(result))
        result = ext_reg.find_components('A', predicate=is_knorg)
        self.assertEqual(1, len(result))
        result = ext_reg.find_extensions('B', predicate=is_knorg)
        self.assertEqual(2, len(result))
        result = ext_reg.find_extensions('C', predicate=is_knorg)
        self.assertEqual(0, len(result))
        result = ext_reg.find_components('C', predicate=is_knorg)
        self.assertEqual(0, len(result))
        result = ext_reg.find_extensions('A', predicate=is_gnatz)
        self.assertEqual(2, len(result))
        result = ext_reg.find_components('A', predicate=is_gnatz)
        self.assertEqual(2, len(result))
        result = ext_reg.find_extensions('B', predicate=is_gnatz)
        self.assertEqual(1, len(result))
        result = ext_reg.find_components('B', predicate=is_gnatz)
        self.assertEqual(1, len(result))
        result = ext_reg.find_extensions('C', predicate=is_gnatz)
        self.assertEqual(0, len(result))
        result = ext_reg.find_components('C', predicate=is_gnatz)
        self.assertEqual(0, len(result))


class TestComponent:
    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs


class ImportTest(unittest.TestCase):
    def test_import_component(self):
        loader = import_component('test.util.test_extension:TestComponent')
        self.assertTrue(callable(loader))
        extension = Extension('test_point', 'test_component', component='dummy')
        component = loader(extension)
        self.assertIs(TestComponent, component)

    def test_import_and_transform_component(self):

        def transform(imported_component: Any, loaded_extension_: Extension):
            return imported_component(-1, name=loaded_extension_.name)

        loader = import_component('test.util.test_extension:TestComponent',
                                  transform=transform)
        self.assertTrue(callable(loader))
        extension = Extension('test_point', 'test_component', component='dummy')
        component = loader(extension)
        self.assertIsInstance(component, TestComponent)
        self.assertEqual((-1,), component.args)
        self.assertEqual({'name': 'test_component'}, component.kwargs)

    def test_import_component_and_call(self):
        loader = import_component('test.util.test_extension:TestComponent', call=True, call_args=[42],
                                  call_kwargs={'help': '!'})
        self.assertTrue(callable(loader))
        extension = Extension('test', 'test', component='x')
        component = loader(extension)
        self.assertIsInstance(component, TestComponent)
        self.assertEqual((42,), component.args)
        self.assertEqual({'help': '!'}, component.kwargs)


# test/util/test_geojson.py

import unittest

from xcube.util.geojson import GeoJSON


class GeoJSONTest(unittest.TestCase):

    def test_is_point(self):
        self.assertTrue(GeoJSON.is_point(dict(type='Point', coordinates=[2.13, 42.2])))
        self.assertFalse(GeoJSON.is_point(dict(type='Feature', properties=None)))

    def test_is_geometry(self):
        self.assertTrue(GeoJSON.is_geometry(dict(type='Point', coordinates=[2.13, 42.2])))
        self.assertTrue(GeoJSON.is_geometry(dict(type='Point', coordinates=None)))
        self.assertFalse(GeoJSON.is_geometry(dict(type='Point')))

        self.assertTrue(GeoJSON.is_geometry(dict(type='GeometryCollection', geometries=None)))
        self.assertTrue(GeoJSON.is_geometry(dict(type='GeometryCollection', geometries=[])))
        self.assertFalse(GeoJSON.is_geometry(dict(type='GeometryCollection')))

        self.assertFalse(GeoJSON.is_geometry(dict(type='Feature', properties=None)))

    def test_is_feature(self):
        self.assertTrue(GeoJSON.is_feature(dict(type='Feature',
                                                geometry=dict(type='Point',
                                                              coordinates=[2.13, 42.2]))))
        self.assertFalse(GeoJSON.is_feature(dict(type='Point', coordinates=[2.13, 42.2])))

    def test_is_feature_collection(self):
        self.assertTrue(GeoJSON.is_feature_collection(dict(type='FeatureCollection',
                                                           features=[dict(type='Feature',
                                                                          geometry=dict(type='Point',
                                                                                        coordinates=[
                                                                                            2.13,
                                                                                            42.2]))])))
        self.assertFalse(GeoJSON.is_feature_collection(dict(type='Point', coordinates=[2.13, 42.2])))

    def test_get_type_name(self):
        self.assertEqual('Feature',
                         GeoJSON.get_type_name(dict(type='Feature')))
        self.assertEqual(None,
                         GeoJSON.get_type_name(dict(pype='Feature')))
        self.assertEqual(None,
                         GeoJSON.get_type_name(dict()))
        self.assertEqual(None,
                         GeoJSON.get_type_name(17))

    def test_get_feature_geometry(self):
        self.assertEqual(dict(type='Point', coordinates=[2.13, 42.2]),
                         GeoJSON.get_feature_geometry(dict(type='Feature',
                                                           geometry=dict(type='Point', coordinates=[2.13, 42.2]))))
        self.assertEqual(None,
                         GeoJSON.get_feature_geometry(dict(type='Pleature',
                                                           geometry=dict(type='Point', coordinates=[2.13, 42.2]))))
        self.assertEqual(None,
                         GeoJSON.get_feature_geometry(dict(type='Feature',
                                                           geometry=dict(type='Point'))))
        self.assertEqual(None,
                         GeoJSON.get_feature_geometry(dict(type='Feature',
                                                           geometry=17)))


# test/util/test_ipython.py

import unittest

from xcube.util.ipython import register_json_formatter


class IPythonTest(unittest.TestCase):

    def test_it(self):
        class _FormatterTest1:
            def to_dict(self):
                return dict()

        # Should work
        register_json_formatter(_FormatterTest1)

        class _FormatterTest2:
            def to_dictomat(self):
                return dict()

        # Should not work
        with self.assertRaises(ValueError) as cm:
            register_json_formatter(_FormatterTest2)
        self.assertTrue(f'{cm.exception}'.endswith("FormatterTest2'> must define a to_dict() method"),
                        msg=f'{cm.exception}')


# test/util/test_jsonschema.py

import unittest
from collections import namedtuple
from typing import Dict, Any

from xcube.util.jsonschema import JsonArraySchema
from xcube.util.jsonschema import JsonBooleanSchema
from xcube.util.jsonschema import JsonComplexSchema
from xcube.util.jsonschema import JsonDateSchema
from xcube.util.jsonschema import JsonDatetimeSchema
from xcube.util.jsonschema import JsonIntegerSchema
from xcube.util.jsonschema import JsonNullSchema
from xcube.util.jsonschema import JsonNumberSchema
from xcube.util.jsonschema import JsonObject
from xcube.util.jsonschema import JsonObjectSchema
from xcube.util.jsonschema import JsonSimpleSchema
from xcube.util.jsonschema import JsonStringSchema


class JsonComplexSchemaTest(unittest.TestCase):

    def test_base_props_validated(self):
        with self.assertRaises(ValueError) as cm:
            JsonComplexSchema()
        self.assertEqual('exactly one of one_of, any_of, all_of must be given',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            JsonComplexSchema(one_of=[JsonStringSchema(), JsonIntegerSchema()],
                              all_of=[JsonStringSchema(), JsonIntegerSchema()])
        self.assertEqual('exactly one of one_of, any_of, all_of must be given',
                         f'{cm.exception}')

    def test_to_dict(self):
        self.assertEqual(
            {'oneOf': [{'multipleOf': 5, 'type': 'integer'},
                       {'multipleOf': 3, 'type': 'integer'}]},
            JsonComplexSchema(one_of=[JsonIntegerSchema(multiple_of=5),
                                      JsonIntegerSchema(multiple_of=3)]).to_dict())
        self.assertEqual(
            {'anyOf': [{'multipleOf': 5, 'type': 'integer'},
                       {'multipleOf': 3, 'type': 'integer'}]},
            JsonComplexSchema(any_of=[JsonIntegerSchema(multiple_of=5),
                                      JsonIntegerSchema(multiple_of=3)]).to_dict())
        self.assertEqual(
            {'allOf': [{'multipleOf': 5, 'type': 'integer'},
                       {'multipleOf': 3, 'type': 'integer'}]},
            JsonComplexSchema(all_of=[JsonIntegerSchema(multiple_of=5),
                                      JsonIntegerSchema(multiple_of=3)]).to_dict())


class JsonSimpleSchemaTest(unittest.TestCase):

    def test_base_props_validated(self):
        with self.assertRaises(ValueError) as cm:
            JsonSimpleSchema('int')
        self.assertEqual('type must be one of "array", "boolean", "integer", "null", "number", "object", "string"',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            JsonSimpleSchema('integer', factory='int')
        self.assertEqual('factory must be callable',
                         f'{cm.exception}')

        with self.assertRaises(ValueError) as cm:
            JsonSimpleSchema('integer', serializer='int')
        self.assertEqual('serializer must be callable',
                         f'{cm.exception}')

    def test_to_dict(self):
        self.assertEqual(
            {
                'type': ['integer', 'null'],
                'title': 'Number of Gnarzes',
                'description': 'Not really required',
                'enum': [10, 100, 1000],
                'default': 10,
                'const': 100,
            },
            JsonSimpleSchema('integer',
                             title='Number of Gnarzes',
                             description='Not really required',
                             enum=[10, 100, 1000],
                             const=100,
                             default=10,
                             nullable=True).to_dict())


class JsonNullSchemaTest(unittest.TestCase):

    def test_to_json_null(self):
        self.assertEqual(None, JsonNullSchema().to_instance(None))

    def test_from_json_null(self):
        self.assertEqual(None, JsonNullSchema().from_instance(None))


class JsonBooleanSchemaTest(unittest.TestCase):

    def test_to_json_boolean(self):
        self.assertEqual(True, JsonBooleanSchema().to_instance(True))

    def test_from_json_boolean(self):
        self.assertEqual(True, JsonBooleanSchema().from_instance(True))


class JsonIntegerSchemaTest(unittest.TestCase):

    def test_to_instance(self):
        self.assertEqual(45, JsonIntegerSchema(minimum=0,
                                               maximum=100,
                                               multiple_of=5).to_instance(45))

    def test_from_instance(self):
        self.assertEqual(45, JsonIntegerSchema(minimum=0,
                                               maximum=100,
                                               multiple_of=5).from_instance(45))


class JsonNumberSchemaTest(unittest.TestCase):
    def test_to_instance(self):
        self.assertEqual(0.6, JsonNumberSchema(exclusive_minimum=0,
                                               exclusive_maximum=1).to_instance(0.6))

    def test_from_instance(self):
        self.assertEqual(6, JsonNumberSchema(minimum=0,
                                             maximum=10,
                                             multiple_of=2).from_instance(6))

    def test_check_type(self):
        with self.assertRaises(ValueError) as cm:
            JsonNumberSchema(type='float')
        self.assertEqual('Type must be one of "integer", "number"', f'{cm.exception}')

    def test_to_dict(self):
        self.assertEqual({'type': 'number'}, JsonNumberSchema().to_dict())
        self.assertEqual({'type': 'number'}, JsonNumberSchema(nullable=False).to_dict())
        self.assertEqual({'type': ['number', 'null']}, JsonNumberSchema(nullable=True).to_dict())
        self.assertEqual({'type': 'number',
                          'exclusiveMinimum': 0,
                          'maximum': 100,
                          'multipleOf': 10},
                         JsonNumberSchema(exclusive_minimum=0,
                                          maximum=100,
                                          multiple_of=10).to_dict())
        self.assertEqual({'type': 'integer',
                          'minimum': 100,
                          'exclusiveMaximum': 200,
                          'multipleOf': 20},
                         JsonIntegerSchema(minimum=100,
                                           exclusive_maximum=200,
                                           multiple_of=20).to_dict())


class JsonStringSchemaTest(unittest.TestCase):
    def test_to_instance(self):
        self.assertEqual('pieps', JsonStringSchema().to_instance('pieps'))
        self.assertEqual('pieps', JsonStringSchema(min_length=0,
                                                   max_length=10).to_instance('pieps'))
        self.assertEqual('pieps', JsonStringSchema(pattern='.*').to_instance('pieps'))
        self.assertEqual('2020-01-03T03:30:01.99+03:30',
                         JsonStringSchema(format='date-time').
                         to_instance('2020-01-03T03:30:01.99+03:30'))

    def test_from_instance(self):
        self.assertEqual('pieps', JsonStringSchema().from_instance('pieps'))

    def test_to_dict(self):
        self.assertEqual({'type': 'string'}, JsonStringSchema().to_dict())
        self.assertEqual({'type': 'string'}, JsonStringSchema(nullable=False).to_dict())
        self.assertEqual({'type': ['string', 'null']}, JsonStringSchema(nullable=True).to_dict())
        self.assertEqual({'type': 'string', 'format': 'uri'}, JsonStringSchema(format='uri').to_dict())


class JsonDateSchemaTest(unittest.TestCase):

    def test_to_instance(self):
        self.assertEqual('2020-06-03',
                         JsonDateSchema(min_date='2020-02-01',
                                        max_date='2020-07-05').to_instance('2020-06-03'))

    def test_to_dict(self):
        self.assertEqual({'type': 'string',
                          'format': 'date',
                          'minDate': '2020-02-01',
                          'maxDate': '2020-07-05'
                          },
                         JsonDateSchema(min_date='2020-02-01',
                                        max_date='2020-07-05').to_dict())

        self.assertEqual({'type': ['string', 'null'],
                          'format': 'date',
                          'minDate': '2020-02-01',
                          'maxDate': '2020-07-05'
                          },
                         JsonDateSchema(min_date='2020-02-01',
                                        max_date='2020-07-05',
                                        nullable=True).to_dict())

    def test_store_date_limits(self):
        minimum = '1981-05-06'
        maximum = '1982-09-15'
        schema = JsonDateSchema(min_date=minimum,
                                max_date=maximum)
        self.assertEqual(minimum, schema.min_date)
        self.assertEqual(maximum, schema.max_date)

    def test_min_max_validity_checks(self):
        with self.assertRaises(ValueError):
            JsonDateSchema(min_date='2002-02-02T10:20:14')
        with self.assertRaises(ValueError):
            JsonDateSchema(max_date='pippo')

    def test_new_range(self):
        self.assertEqual(
            {'type': 'array',
             'items': [{'type': 'string',
                        'format': 'date',
                        },
                       {'type': 'string',
                        'format': 'date',
                        }
                       ],
             },
            JsonDateSchema.new_range().to_dict())

        self.assertEqual(
            {'type': ['array', 'null'],
             'items': [{'type': ['string', 'null'],
                        'format': 'date',
                        'minDate': '2020-02-01',
                        'maxDate': '2020-07-05',
                        },
                       {'type': ['string', 'null'],
                        'format': 'date',
                        'minDate': '2020-02-01',
                        'maxDate': '2020-07-05',
                        }
                       ],
             },
            JsonDateSchema.new_range(min_date='2020-02-01',
                                     max_date='2020-07-05',
                                     nullable=True).to_dict())


class JsonDatetimeSchemaTest(unittest.TestCase):

    def test_to_instance(self):
        self.assertEqual('2020-06-12T12:30:19Z',
                         JsonDatetimeSchema(min_datetime='2020-02-01T00:00:00Z',
                                            max_datetime='2020-07-05T00:00:00Z').to_instance(
                             '2020-06-12T12:30:19Z'))

    def test_to_dict(self):
        self.assertEqual({'type': 'string',
                          'format': 'date-time',
                          'minDatetime': '2020-02-01T00:00:00Z',
                          'maxDatetime': '2020-07-05T00:00:00Z'},
                         JsonDatetimeSchema(min_datetime='2020-02-01T00:00:00Z',
                                            max_datetime='2020-07-05T00:00:00Z').to_dict())

    def test_store_date_limits(self):
        minimum = '1981-05-06T00:00:00+00:00'
        maximum = '1982-09-15T00:00:00+00:00'
        schema = JsonDatetimeSchema(min_datetime=minimum,
                                    max_datetime=maximum)
        self.assertEqual(minimum, schema.min_datetime)
        self.assertEqual(maximum, schema.max_datetime)

    def test_min_max_validity_checks(self):
        with self.assertRaises(ValueError):
            JsonDatetimeSchema(  # missing timezone specifier
                min_datetime='1980-02-03T12:34:56',
                max_datetime='1982-02-03T23:34:56+05:00')
        with self.assertRaises(ValueError):
            JsonDatetimeSchema(min_datetime='1980-02-03T12:34:56-08:00',
                               # invalid date
                               max_datetime='1985-01-32')

    def test_new_range(self):
        self.assertEqual(
            {'type': 'array',
             'items': [{'type': 'string',
                        'format': 'date-time',
                        },
                       {'type': 'string',
                        'format': 'date-time',
                        }
                       ],
             },
            JsonDatetimeSchema.new_range().to_dict())

        self.assertEqual(
            {'type': ['array', 'null'],
             'items': [{'type': ['string', 'null'],
                        'format': 'date-time',
                        'minDatetime': '2002-01-01T00:00:00Z',
                        'maxDatetime': '2020-01-01T00:00:00Z',
                        },
                       {'type': ['string', 'null'],
                        'format': 'date-time',
                        'minDatetime': '2002-01-01T00:00:00Z',
                        'maxDatetime': '2020-01-01T00:00:00Z',
                        }
                       ],
             },
            JsonDatetimeSchema.new_range(min_datetime='2002-01-01T00:00:00Z',
                                         max_datetime='2020-01-01T00:00:00Z',
                                         nullable=True).to_dict())


class JsonArraySchemaTest(unittest.TestCase):

    def test_to_instance(self):
        self.assertEqual([False, 2, 'U'],
                         JsonArraySchema().to_instance([False, 2, 'U']))

    def test_to_instance_tuple(self):
        self.assertEqual([False, 2, 'U'],
                         JsonArraySchema(items=[JsonBooleanSchema(),
                                                JsonIntegerSchema(),
                                                JsonStringSchema()]).to_instance([False, 2, 'U']))

    def test_from_instance(self):
        self.assertEqual([False, 2, 'U'],
                         JsonArraySchema().from_instance([False, 2, 'U']))

    def test_from_instance_tuple(self):
        self.assertEqual([False, 2, 'U'],
                         JsonArraySchema(items=[JsonBooleanSchema(),
                                                JsonIntegerSchema(),
                                                JsonStringSchema()]).from_instance([False, 2, 'U']))

    def test_tuple_validates_as_array(self):
        self.assertTupleEqual((1, 2, 3),
                              JsonArraySchema().to_instance((1, 2, 3)))

    def test_from_instance_array_object(self):
        value = [{'name': 'Bibo', 'age': 15},
                 {'name': 'Ernie', 'age': 12}]

        person_schema = JsonObjectSchema(
            properties=dict(name=JsonStringSchema(), age=JsonIntegerSchema(),
                            deleted=JsonBooleanSchema(default=False)))
        schema = JsonArraySchema(items=person_schema)
        self.assertEqual([{'name': 'Bibo', 'age': 15, 'deleted': False},
                          {'name': 'Ernie', 'age': 12, 'deleted': False}],
                         schema.from_instance(value))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])
        person_schema.factory = Person
        self.assertEqual([Person(name='Bibo', age=15, deleted=False),
                          Person(name='Ernie', age=12, deleted=False)],
                         schema.from_instance(value))

        Assignment = namedtuple('Assignment', ['persons'])

        def factory(persons):
            return Assignment(persons=persons)

        schema.factory = factory
        self.assertEqual(Assignment(persons=[Person(name='Bibo', age=15, deleted=False),
                                             Person(name='Ernie', age=12, deleted=False)]),
                         schema.from_instance(value))


class JsonObjectSchemaTest(unittest.TestCase):

    def test_from_json_object(self):
        value = {'name': 'Bibo', 'age': 12, 'deleted': True}

        person_schema = JsonObjectSchema(properties=dict(name=JsonStringSchema(),
                                                         age=JsonIntegerSchema(),
                                                         deleted=JsonBooleanSchema(default=False)))
        self.assertEqual(value,
                         person_schema.from_instance(value))

        self.assertEqual({'name': 'Bibo', 'age': 12, 'deleted': False},
                         person_schema.from_instance({'name': 'Bibo', 'age': 12}))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])
        person_schema.factory = Person
        self.assertEqual(Person(name='Bibo', age=12, deleted=True),
                         person_schema.from_instance(value))

    def test_to_json_object(self):
        person_schema = JsonObjectSchema(properties=dict(name=JsonStringSchema(),
                                                         age=JsonIntegerSchema(),
                                                         deleted=JsonBooleanSchema(default=False)))

        value = {'name': 'Bibo', 'age': 12, 'deleted': True}

        self.assertEqual(value,
                         person_schema.to_instance(value))

        # ok, because person_schema does not explicitly say additional_properties=False
        value_extra = {'name': 'Bibo', 'age': 12, 'deleted': True, 'comment': 'Hello!'}
        self.assertEqual(value_extra,
                         person_schema.to_instance(value_extra))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])

        def serialize(person: Person) -> Dict[str, Any]:
            return person._asdict()

        person_schema.serializer = serialize

        person = Person(**value)
        self.assertEqual(value,
                         person_schema.to_instance(person))

    def test_from_json_object_object(self):
        person_schema = JsonObjectSchema(properties=dict(name=JsonStringSchema(),
                                                         age=JsonIntegerSchema(),
                                                         deleted=JsonBooleanSchema(default=False)))
        schema = JsonObjectSchema(properties=dict(person=person_schema))

        value = {'person': {'name': 'Bibo', 'age': 15}}

        self.assertEqual({'person': {'name': 'Bibo', 'age': 15, 'deleted': False}},
                         schema.from_instance(value))

        Assignment = namedtuple('Assignment', ['person'])
        schema.factory = Assignment
        self.assertEqual(Assignment(person={'name': 'Bibo', 'age': 15, 'deleted': False}),
                         schema.from_instance(value))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])
        person_schema.factory = Person
        self.assertEqual(Assignment(person=Person(name='Bibo', age=15, deleted=False)),
                         schema.from_instance(value))

    def test_from_json_object_array_object(self):
        person_schema = JsonObjectSchema(properties=dict(name=JsonStringSchema(),
                                                         age=JsonIntegerSchema(),
                                                         deleted=JsonBooleanSchema(default=False)))

        schema = JsonObjectSchema(properties=dict(persons=JsonArraySchema(items=person_schema)))

        value = {'persons': [{'name': 'Bibo', 'age': 15},
                             {'name': 'Ernie', 'age': 12}]}

        self.assertEqual({'persons': [{'name': 'Bibo', 'age': 15, 'deleted': False},
                                      {'name': 'Ernie', 'age': 12, 'deleted': False}]},
                         schema.from_instance(value))

        Assignment = namedtuple('Assignment', ['persons'])
        schema.factory = Assignment
        self.assertEqual(Assignment(persons=[{'name': 'Bibo', 'age': 15, 'deleted': False},
                                             {'name': 'Ernie', 'age': 12, 'deleted': False}]),
                         schema.from_instance(value))

        Person = namedtuple('Person', ['name', 'age', 'deleted'])
        person_schema.factory = Person
        self.assertEqual(Assignment(persons=[Person(name='Bibo', age=15, deleted=False),
                                             Person(name='Ernie', age=12, deleted=False)]),
                         schema.from_instance(value))

    def test_from_json_object_additional_properties_is_schema(self):
        Person = namedtuple('Person', ['name', 'age', 'deleted'])

        person_schema = JsonObjectSchema(
            properties=dict(
                name=JsonStringSchema(),
                age=JsonIntegerSchema(),
                deleted=JsonBooleanSchema(default=False)
            ),
            factory=Person,
        )

        schema = JsonObjectSchema(
            additional_properties=person_schema,
        )

        value = {
            'p1': {'name': 'Bibo', 'age': 15, 'deleted': True},
            'p2': {'name': 'Ernie', 'age': 12, 'deleted': False},
        }

        self.assertEqual(
            {
                'p1': Person(name='Bibo', age=15, deleted=True),
                'p2': Person(name='Ernie', age=12, deleted=False),
            },
            schema.from_instance(value))

    def test_process_kwargs_subset(self):
        schema = JsonObjectSchema(
            properties=dict(
                client_id=JsonStringSchema(default='bibo'),
                client_secret=JsonStringSchema(default='2w908435t'),
                geom=JsonStringSchema(),
                crs=JsonStringSchema(const='WGS84'),
                spatial_res=JsonNumberSchema(),
                time_range=JsonStringSchema(),
                time_period=JsonStringSchema(default='8D'),
                max_cache_size=JsonIntegerSchema(),
            ),
            required=['client_id', 'client_secret', 'geom', 'crs', 'spatial_res', 'time_range'],
        )

        kwargs = dict(client_secret='094529g',
                      geom='POINT (12.2, 53.9)',
                      spatial_res=0.5,
                      time_range='2010,2014',
                      max_cache_size=2 ** 32)

        cred_kwargs, kwargs = schema.process_kwargs_subset(kwargs, ['client_id', 'client_secret'])
        self.assertEqual(dict(client_id='bibo', client_secret='094529g'),
                         cred_kwargs)
        self.assertEqual(dict(geom='POINT (12.2, 53.9)',
                              spatial_res=0.5,
                              time_range='2010,2014',
                              max_cache_size=2 ** 32),
                         kwargs)

        ds_kwargs, kwargs = schema.process_kwargs_subset(kwargs, ['geom', 'crs',
                                                                  'spatial_res', 'time_range', 'time_period'])
        self.assertEqual(dict(crs='WGS84',
                              geom='POINT (12.2, 53.9)',
                              spatial_res=0.5,
                              time_range='2010,2014'),
                         ds_kwargs)
        self.assertEqual(dict(max_cache_size=2 ** 32),
                         kwargs)

    def test_to_dict(self):
        schema = JsonObjectSchema(
            properties=dict(
                consolidated=JsonBooleanSchema()
            )
        )
        self.assertEqual(
            {
                'type': 'object',
                'properties': {
                    'consolidated': {
                        'type': 'boolean'
                    }
                },
            },
            schema.to_dict()
        )


class IHasAProperty(JsonObject):
    def __init__(self, name, age):
        # A private state property
        self._name = name
        # A state property
        self.age = age

    @property
    def id(self):
        # A derived property
        return f'{self.name}_{self.age}'

    @property
    def name(self):
        # A state property
        return self._name

    @classmethod
    def get_schema(cls):
        return JsonObjectSchema(
            properties=dict(
                name=JsonStringSchema(),
                age=JsonIntegerSchema()
            ),
            required=['name', 'age'],
            factory=cls
        )


class IHasAProperty2(IHasAProperty):
    @classmethod
    def get_schema(cls):
        schema = super().get_schema()
        schema.additional_properties = False
        return schema


class JsonObjectWithPropertyTest(unittest.TestCase):
    def test_with_additional_properties(self):
        obj_dict = {'name': 'bibo', 'age': 2021 - 1969}
        obj = IHasAProperty.from_dict(obj_dict)
        actual_dict = obj.to_dict()
        self.assertEqual({'age': 52, 'id': 'bibo_52', 'name': 'bibo'}, actual_dict)

    def test_without_additional_properties(self):
        obj_dict = {'name': 'bibo', 'age': 2021 - 1969}
        obj = IHasAProperty2.from_dict(obj_dict)
        actual_dict = obj.to_dict()
        self.assertEqual(obj_dict, actual_dict)


# test/util/test_ndarray_to_cmap_image.py

import copy
from unittest import TestCase

import numpy as np
from PIL import Image
from matplotlib import cm


# * http://stackoverflow.com/questions/14448763/is-there-a-convenient-way-to-apply-a-lookup-table-to-a-large-array-in-numpy
# * http://stackoverflow.com/questions/10965417/how-to-convert-numpy-array-to-pil-image-applying-matplotlib-colormap
# * http://stackoverflow.com/questions/10443295/combine-3-separate-numpy-arrays-to-an-rgb-image-in-python
# * http://stackoverflow.com/questions/2578752/how-can-i-plot-nan-values-as-a-special-color-with-imshow-in-matplotlib
#
# * http://matplotlib.org/api/cm_api.html#module-matplotlib.cm
# * http://matplotlib.org/users/colormaps.html
# * http://matplotlib.org/api/colors_api.html#
# * https://bids.github.io/colormap/

class PILImageTest(TestCase):
    def test_array_to_rgba(self):
        nan = np.nan
        array = np.array([
            [0.0, 0.1, 0.2, nan],
            [0.2, 0.3, nan, 0.5],
            [0.4, nan, 0.6, 0.7],
            [nan, 0.7, 0.8, 0.9],
        ])
        # print(array, array.dtype)

        min = 0.1
        max = 0.8

        array.clip(min, max, out=array)
        # print(array, array.dtype)

        array -= min
        array *= 1.0 / (max - min)
        # print(array, array.dtype)

        array = np.ma.masked_invalid(array, copy=False)
        # print(array, array.dtype)

        # import pprint
        # pprint.pprint(dir(cm.gist_earth))

        # We want to have transparency where the array is masked out.
        # This is done via colormap.set_bad(color, alpha).
        # However, in doing we would modify the global cm.gist_earth singleton so we create a copy here:
        #
        cmap_copy = copy.copy(cm.gist_earth)
        cmap_copy.set_bad('k', 0)
        array = cmap_copy(array, bytes=True)
        # print(array, array.dtype)
        self.assertEqual([[[0, 0, 0, 255],
                           [0, 0, 0, 255],
                           [23, 64, 121, 255],
                           [0, 0, 0, 0],
                           ],

                          [[23, 64, 121, 255],
                           [48, 128, 126, 255],
                           [0, 0, 0, 0],
                           [130, 168, 83, 255],
                           ],

                          [[65, 149, 82, 255],
                           [0, 0, 0, 0],
                           [184, 179, 95, 255],
                           [212, 176, 147, 255],
                           ],

                          [[0, 0, 0, 0],
                           [212, 176, 147, 255],
                           [253, 250, 250, 255],
                           [253, 250, 250, 255],
                           ],
                          ], array.tolist())

        im = Image.fromarray(array, mode='RGBA')

        # im.show('Raw Image')

        self.assertEqual((0, 0, 0, 255), im.getpixel((0, 0)))
        self.assertEqual((23, 64, 121, 255), im.getpixel((2, 0)))
        self.assertEqual((0, 0, 0, 0), im.getpixel((1, 2)))
        self.assertEqual((212, 176, 147, 255), im.getpixel((1, 3)))


# test/util/test_perf.py

import time
from unittest import TestCase

from xcube.util.perf import measure_time_cm


class MockLogger:
    def __init__(self):
        self.output = []

    # noinspection PyUnusedLocal
    def info(self, msg: str, *args, **kwargs):
        if len(args) == 1:
            args = args[0]
        self.output.append(msg % args)


class MeasureTimeTest(TestCase):
    def test_enabled(self):
        logger = MockLogger()
        measure_time = measure_time_cm(disabled=False, logger=logger)

        with measure_time("Hello") as cm:
            time.sleep(0.06)
        self.assertTrue(hasattr(cm, "duration"))
        self.assertTrue(cm.duration > 0.05)
        self.assertIs(logger, cm.logger)
        self.assertEqual("Hello", cm.message)
        self.assertEqual(1, len(logger.output))
        self.assertTrue(logger.output[0].startswith("Hello: took "))

        with measure_time("Hello %s", "Mrs X") as cm:
            time.sleep(0.06)
        self.assertTrue(hasattr(cm, "duration"))
        self.assertTrue(cm.duration > 0.05)
        self.assertIsNotNone(cm.logger)
        self.assertEqual("Hello %s", cm.message)
        self.assertEqual(("Mrs X",), cm.args)
        self.assertEqual(2, len(logger.output))
        self.assertTrue(logger.output[1].startswith("Hello Mrs X: took "))

    def test_enabled_deprecated(self):
        measure_time = measure_time_cm(disabled=False)
        with measure_time(tag="Hello") as cm:
            time.sleep(0.06)
        self.assertTrue(hasattr(cm, "duration"))
        self.assertTrue(cm.duration > 0.05)
        self.assertIsNotNone(cm.logger)
        self.assertEqual("Hello", cm.message)

    def test_disabled(self):
        measure_time = measure_time_cm(disabled=True)
        with measure_time("hello") as cm:
            time.sleep(0.05)
        self.assertTrue(hasattr(cm, "duration"))
        self.assertIsNone(cm.duration)
        self.assertIsNone(cm.logger)


# test/util/test_plugin.py

import unittest

from xcube.util.extension import ExtensionRegistry
from xcube.util.plugin import get_plugins, load_plugins, discover_plugin_modules


def init_plugin(ext_registry: ExtensionRegistry):
    """A test plugin that registers test extensions"""
    ext_registry.add_extension(component=object(), point='test.util.test_plugin', name='ext1')
    ext_registry.add_extension(component=object(), point='test.util.test_plugin', name='ext2')
    ext_registry.add_extension(component=object(), point='test.util.test_plugin', name='ext3')


def init_plugin_bad():
    raise RuntimeError()


class EntryPoint:
    def __init__(self, name, func):
        self.name = name
        self.func = func

    def load(self):
        return self.func


class EntryPointBad(EntryPoint):

    def load(self):
        raise RuntimeError()


class PluginTest(unittest.TestCase):

    def setUp(self):
        self.ext_registry = ExtensionRegistry()

    def test_get_xcube_default_plugins(self):
        plugins = get_plugins()
        self.assertIsNotNone(plugins)
        self.assertIn('xcube', plugins)

    def test_load_plugins_by_entry_points(self):
        plugins = load_plugins([EntryPoint('test', init_plugin)], ext_registry=self.ext_registry)
        self.assertEqual(dict(test=dict(name='test', doc='A test plugin that registers test extensions')), plugins)
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext1'))
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext2'))
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext3'))

    def test_load_plugins_by_module_discovery(self):
        entry_points = discover_plugin_modules(module_prefixes=['test'])
        plugins = load_plugins(entry_points, ext_registry=self.ext_registry)
        self.assertEqual(dict(test=dict(name='test', doc='A test plugin that registers test extensions')), plugins)
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext1'))
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext2'))
        self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext3'))

    def test_load_plugins_by_bad_entry_point(self):
        plugins = load_plugins([EntryPointBad('test', init_plugin)], ext_registry=self.ext_registry)
        self.assertEqual({}, plugins)
        self.assertEqual([], self.ext_registry.find_components('test.util.test_plugin'))

    def test_load_plugins_by_bad_init_plugin(self):
        plugins = load_plugins([EntryPoint('test', init_plugin_bad)], ext_registry=self.ext_registry)
        self.assertEqual({}, plugins)
        self.assertEqual([], self.ext_registry.find_components('test.util.test_plugin'))

    def test_load_plugins_init_plugin_not_callable(self):
        plugins = load_plugins([EntryPoint('test', "init_plugin_not_callable")], ext_registry=self.ext_registry)
        self.assertEqual({}, plugins)
        self.assertEqual([], self.ext_registry.find_components('test'))

    def test_load_plugins_by_failing_module_discovery(self):
        entry_points = discover_plugin_modules(module_prefixes=['random'])
        plugins = load_plugins(entry_points, ext_registry=self.ext_registry)
        self.assertEqual({}, plugins)
        self.assertEqual([], self.ext_registry.find_components('test.util.test_plugin'))


# test/util/test_progress.py

import unittest
from typing import Sequence

import dask
import dask.array.random

from xcube.util.progress import ProgressObserver
from xcube.util.progress import ProgressState
from xcube.util.progress import add_progress_observers
from xcube.util.progress import new_progress_observers
from xcube.util.progress import observe_dask_progress
from xcube.util.progress import observe_progress


class TestProgressObserver(ProgressObserver):
    def __init__(self, record_errors=False):
        self.record_errors = record_errors
        self.calls = []

    def on_begin(self, state_stack: Sequence[ProgressState]):
        self.calls.append(('begin', self._serialize_stack(state_stack)))

    def on_update(self, state_stack: Sequence[ProgressState]):
        self.calls.append(('update', self._serialize_stack(state_stack)))

    def on_end(self, state_stack: Sequence[ProgressState]):
        self.calls.append(('end', self._serialize_stack(state_stack)))

    def _serialize_stack(self, state_stack):
        if self.record_errors:
            return [(s.label, s.progress, s.finished, s.exc_info_text) for s in state_stack]
        else:
            return [(s.label, s.progress, s.finished) for s in state_stack]


class ObserveProgressTest(unittest.TestCase):

    def test_observe_progress(self):
        observer = TestProgressObserver()
        observer.activate()

        with observe_progress('computing', 4) as reporter:
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that takes 2 units
            reporter.worked(2)

        self.assertIsInstance(reporter.state, ProgressState)
        self.assertIsInstance(reporter.state.total_time, float)
        self.assertTrue(reporter.state.total_time >= 0.0)

        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False)]),
                ('update', [('computing', 0.25, False)]),
                ('update', [('computing', 0.5, False)]),
                ('update', [('computing', 1.0, False)]),
                ('end', [('computing', 1.0, True)])
            ],
            observer.calls)

    def test_nested_observe_progress(self):
        observer = TestProgressObserver()
        observer.deactivate()
        observer.activate()

        with observe_progress('computing', 4) as reporter:
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that will take 2 units
            reporter.will_work(2)
            with observe_progress('loading', 4) as nested_reporter:
                # do something that takes 3 units
                nested_reporter.worked(3)
                # do something that takes 1 unit
                nested_reporter.worked(1)

        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False)]),
                ('update', [('computing', 0.25, False)]),
                ('update', [('computing', 0.5, False)]),
                ('begin', [('computing', 0.5, False), ('loading', 0.0, False)]),
                ('update', [('computing', 0.875, False), ('loading', 0.75, False)]),
                ('update', [('computing', 1.0, False), ('loading', 1.0, False)]),
                ('end', [('computing', 1.0, False), ('loading', 1.0, True)]),
                ('end', [('computing', 1.0, True)])
            ],
            observer.calls)

    def test_nested_observe_progress_with_new_progress_observers(self):
        observer = TestProgressObserver()
        observer.activate()

        nested_observer = TestProgressObserver()

        with observe_progress('computing', 4) as progress_reporter:
            # do something that takes 1 unit
            progress_reporter.worked(1)
            # do something that takes 1 unit
            progress_reporter.worked(1)
            with new_progress_observers(nested_observer):
                with observe_progress('loading', 4) as progress_reporter_2:
                    # do something that takes 3 units
                    progress_reporter_2.worked(3)
                    # do something that takes 1 unit
                    progress_reporter_2.worked(1)

            # do something that takes 1 unit
            progress_reporter.worked(2)

        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False)]),
                ('update', [('computing', 0.25, False)]),
                ('update', [('computing', 0.5, False)]),
                ('update', [('computing', 1.0, False)]),
                ('end', [('computing', 1.0, True)])
            ],
            observer.calls)

        self.assertEqual(
            [
                ('begin', [('loading', 0.0, False)]),
                ('update', [('loading', 0.75, False)]),
                ('update', [('loading', 1.0, False)]),
                ('end', [('loading', 1.0, True)])
            ],
            nested_observer.calls)

    def test_nested_observe_progress_with_add_progress_observers(self):
        observer = TestProgressObserver()
        observer.activate()

        nested_observer = TestProgressObserver()

        with observe_progress('computing', 4) as reporter:
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that takes 1 unit
            reporter.worked(1)
            # do something that will take 2 units
            reporter.will_work(2)
            with add_progress_observers(nested_observer):
                with observe_progress('loading', 4) as nested_reported:
                    # do something that takes 3 units
                    nested_reported.worked(3)
                    # do something that takes 1 unit
                    nested_reported.worked(1)

        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False)]),
                ('update', [('computing', 0.25, False)]),
                ('update', [('computing', 0.5, False)]),
                ('begin', [('computing', 0.5, False), ('loading', 0.0, False)]),
                ('update', [('computing', 0.875, False), ('loading', 0.75, False)]),
                ('update', [('computing', 1.0, False), ('loading', 1.0, False)]),
                ('end', [('computing', 1.0, False), ('loading', 1.0, True)]),
                ('end', [('computing', 1.0, True)])
            ],
            observer.calls)

        self.assertEqual(
            [
                ('begin', [('computing', 0.5, False), ('loading', 0.0, False)]),
                ('update', [('computing', 0.875, False), ('loading', 0.75, False)]),
                ('update', [('computing', 1.0, False), ('loading', 1.0, False)]),
                ('end', [('computing', 1.0, False), ('loading', 1.0, True)])
            ],
            nested_observer.calls)

    def test_nested_observe_progress_with_exception(self):
        observer = TestProgressObserver(record_errors=True)
        observer.activate()

        try:
            with observe_progress('computing', 10) as reporter:
                # do something that takes 1 unit
                reporter.worked(1)
                # do something that takes 1 unit
                reporter.worked(1)
                # do something that will take 2 units
                reporter.will_work(8)
                with observe_progress('loading', 100) as nested_reported:
                    # do something that takes 3 units
                    nested_reported.worked(15)
                    # now - BANG!
                    raise ValueError('Failed to load')
        except ValueError:
            pass
        self.assertEqual(7, len(observer.calls))
        self.assertEqual(
            [
                ('begin', [('computing', 0.0, False, None)]),
                ('update', [('computing', 0.1, False, None)]),
                ('update', [('computing', 0.2, False, None)]),
                ('begin', [('computing', 0.2, False, None), ('loading', 0.0, False, None)]),
                ('update', [('computing', 0.32, False, None), ('loading', 0.15, False, None)]),
            ],
            observer.calls[0: -2])

        self.assertEqual(2, len(observer.calls[-2]))
        event, states = observer.calls[-2]
        self.assertEqual('end', event)
        self.assertEqual(2, len(states))
        self.assertEqual(4, len(states[0]))
        self.assertEqual(4, len(states[1]))
        self.assertEqual(('computing', 0.32, False), states[0][0:-1])
        self.assertEqual(('loading', 0.15, True), states[1][0:-1])
        error = states[0][-1]
        self.assertIsNone(error)
        error = states[1][-1]
        self.assertIsInstance(error, tuple)
        exc_type, exc_value, exc_traceback = error
        self.assertEqual('ValueError', exc_type)
        self.assertEqual('Failed to load', exc_value)
        self.assertIsInstance(exc_traceback, list)

        self.assertEqual(2, len(observer.calls[-1]))
        event, states = observer.calls[-1]
        self.assertEqual('end', event)
        self.assertEqual(1, len(states))
        self.assertEqual(4, len(states[0]))
        self.assertEqual(('computing', 0.32, True), states[0][0:-1])
        error = states[0][-1]
        self.assertIsInstance(error, tuple)
        exc_type, exc_value, exc_traceback = error
        self.assertEqual('ValueError', exc_type)
        self.assertEqual('Failed to load', exc_value)
        self.assertIsInstance(exc_traceback, list)

    def test_dask_progress(self):
        observer = TestProgressObserver(record_errors=True)
        observer.activate()

        res = dask.array.random.normal(size=(100, 200), chunks=(25, 50))
        with observe_dask_progress('computing', 100):
            res.compute()

        self.assertEqual(4, len(res.chunks[0]))
        self.assertTrue(len(observer.calls) >= 3)
        self.assertEqual(('begin', [('computing', 0.0, False, None)]), observer.calls[0])
        self.assertEqual(('update', [('computing', 5 / 16, False, None)]), observer.calls[5])
        self.assertEqual(('end', [('computing', 15 / 16, True, None)]), observer.calls[-1])


class ProgressStateTest(unittest.TestCase):

    def test_progress_state_props(self):
        state = ProgressState('computing', 100, 3)
        self.assertEqual('computing', state.label)
        self.assertEqual(100, state.total_work)
        self.assertEqual(3, state.super_work)
        self.assertEqual(1, state.super_work_ahead)
        self.assertEqual(0, state.completed_work)
        self.assertEqual(0, state.progress)
        self.assertEqual(0.6, state.to_super_work(20))
        state.inc_work(15)
        self.assertEqual(15, state.completed_work)
        self.assertEqual(0.15, state.progress)
        self.assertEqual(0.6, state.to_super_work(20))


# test/util/test_temp.py

import os.path
import unittest

from xcube.util.temp import new_temp_dir
from xcube.util.temp import new_temp_file
from xcube.util.temp import remove_dir_later
from xcube.util.temp import remove_file_later


class TempTest(unittest.TestCase):
    def test_new_temp_dir(self):
        dir_path = new_temp_dir()
        self.assertIsInstance(dir_path, str)
        self.assertTrue(os.path.basename(dir_path).startswith('xcube-'))
        self.assertTrue(os.path.isabs(dir_path))
        self.assertTrue(os.path.isdir(dir_path))

        dir_path = new_temp_dir(prefix='bibo.', suffix='.zarr')
        self.assertIsInstance(dir_path, str)
        self.assertTrue(os.path.basename(dir_path).startswith('bibo.'))
        self.assertTrue(os.path.basename(dir_path).endswith('.zarr'))
        self.assertTrue(os.path.isabs(dir_path))
        self.assertTrue(os.path.isdir(dir_path))

    def test_new_temp_file(self):
        fd, file_path = new_temp_file()
        self.assertIsInstance(fd, int)
        self.assertIsInstance(file_path, str)
        self.assertTrue(os.path.basename(file_path).startswith('xcube-'))
        self.assertTrue(os.path.isabs(file_path))
        self.assertTrue(os.path.isfile(file_path))

        fd, file_path = new_temp_file(prefix='bibo.', suffix='.zip')
        self.assertIsInstance(fd, int)
        self.assertIsInstance(file_path, str)
        self.assertTrue(os.path.basename(file_path).startswith('bibo.'))
        self.assertTrue(os.path.basename(file_path).endswith('.zip'))
        self.assertTrue(os.path.isabs(file_path))
        self.assertTrue(os.path.isfile(file_path))

    def test_remove_file_later(self):
        with open('__test__.txt', 'w') as stream:
            stream.write('Hello!')
        file_path = remove_file_later('__test__.txt')
        self.assertTrue(os.path.isabs(file_path))

    def test_remove_dir_later(self):
        os.mkdir('__test__')
        dir_path = remove_dir_later('__test__')
        self.assertTrue(os.path.isabs(dir_path))


# test/util/test_tiledimage.py

from unittest import TestCase

import numpy as np
import PIL.Image

from xcube.util.tiledimage import SourceArrayImage
from xcube.util.tiledimage import ColorMappedRgbaImage
from xcube.util.tiledimage import DirectRgbaImage
from xcube.util.tiledimage import OpImage
from xcube.util.tiledimage import NormalizeArrayImage
from xcube.util.tiledimage import trim_tile


class MyTiledImage(OpImage):
    def __init__(self, size, tile_size):
        super().__init__(size, tile_size, (size[0] // tile_size[0], size[1] // tile_size[1]),
                         mode='int32', format='ndarray')

    def compute_tile(self, tile_x, tile_y, rectangle):
        w, h = self.size
        x, y, tw, th = rectangle
        fill_value = float(x + y * w) / float(w * h)
        return np.full((th, tw), fill_value, np.float32)


class ColorMappedRgbaImageTest(TestCase):
    def test_default(self):
        a = np.linspace(0, 255, 24, dtype=np.int32).reshape((4, 6))
        source_image = SourceArrayImage(a, (2, 2))
        cm_rgb_image = ColorMappedRgbaImage(source_image, format='PNG')
        self.assertEqual(cm_rgb_image.size, (6, 4))
        self.assertEqual(cm_rgb_image.tile_size, (2, 2))
        self.assertEqual(cm_rgb_image.num_tiles, (3, 2))
        tile = cm_rgb_image.get_tile(0, 0)
        self.assertIsInstance(tile, PIL.Image.Image)


class DirectRgbaImageTest(TestCase):
    def test_default(self):
        a = np.linspace(0, 255, 24, dtype=np.int32).reshape((4, 6))
        b = np.linspace(255, 0, 24, dtype=np.int32).reshape((4, 6))
        c = np.linspace(50, 200, 24, dtype=np.int32).reshape((4, 6))
        source_images = [
            SourceArrayImage(a, (2, 2)),
            SourceArrayImage(b, (2, 2)),
            SourceArrayImage(c, (2, 2)),
        ]
        cm_rgb_image = DirectRgbaImage(source_images, format='PNG')
        self.assertEqual(cm_rgb_image.size, (6, 4))
        self.assertEqual(cm_rgb_image.tile_size, (2, 2))
        self.assertEqual(cm_rgb_image.num_tiles, (3, 2))
        tile = cm_rgb_image.get_tile(0, 0)
        self.assertIsInstance(tile, PIL.Image.Image)


class NormalizeArrayImageTest(TestCase):
    def test_default(self):
        a = np.arange(0, 24, dtype=np.int32).reshape((4, 6))
        source_image = SourceArrayImage(a, (2, 2))
        target_image = NormalizeArrayImage(source_image)

        self.assertEqual(target_image.size, (6, 4))
        self.assertEqual(target_image.tile_size, (2, 2))
        self.assertEqual(target_image.num_tiles, (3, 2))

        self.assertEqual(target_image.get_tile(0, 0).tolist(), [[0, 1],
                                                                [6, 7]])
        self.assertEqual(target_image.get_tile(1, 0).tolist(), [[2, 3],
                                                                [8, 9]])
        self.assertEqual(target_image.get_tile(2, 0).tolist(), [[4, 5],
                                                                [10, 11]])

        self.assertEqual(target_image.get_tile(0, 1).tolist(), [[12, 13],
                                                                [18, 19]])
        self.assertEqual(target_image.get_tile(1, 1).tolist(), [[14, 15],
                                                                [20, 21]])
        self.assertEqual(target_image.get_tile(2, 1).tolist(), [[16, 17],
                                                                [22, 23]])

    def test_force_2d(self):
        a = np.arange(0, 48, dtype=np.int32).reshape((2, 4, 6))
        source_image = SourceArrayImage(a[0], (2, 2))
        target_image = NormalizeArrayImage(source_image, force_2d=True)

        self.assertEqual(target_image.size, (6, 4))
        self.assertEqual(target_image.tile_size, (2, 2))
        self.assertEqual(target_image.num_tiles, (3, 2))


class SourceArrayImageTest(TestCase):

    def test_flip_y_tiled_evenly(self):
        a = np.arange(0, 24, dtype=np.int32).reshape((4, 6))
        source_array_image = SourceArrayImage(a, (2, 2), flip_y=True)

        self.assertEqual(source_array_image.size, (6, 4))
        self.assertEqual(source_array_image.tile_size, (2, 2))
        self.assertEqual(source_array_image.num_tiles, (3, 2))

        self.assertEqual(source_array_image.get_tile(0, 0).tolist(),
                         [[18, 19],
                          [12, 13]])
        self.assertEqual(source_array_image.get_tile(1, 0).tolist(),
                         [[20, 21],
                          [14, 15]])
        self.assertEqual(source_array_image.get_tile(2, 0).tolist(),
                         [[22, 23],
                          [16, 17]])

        self.assertEqual(source_array_image.get_tile(0, 1).tolist(),
                         [[6, 7],
                          [0, 1]])
        self.assertEqual(source_array_image.get_tile(1, 1).tolist(),
                         [[8, 9],
                          [2, 3]])
        self.assertEqual(source_array_image.get_tile(2, 1).tolist(),
                         [[10, 11],
                          [4, 5]])

    def test_flip_y_not_tiled_evenly(self):
        a = np.arange(0, 30, dtype=np.int32).reshape((5, 6))
        source_array_image = SourceArrayImage(a, (2, 2), flip_y=True)

        self.assertEqual(source_array_image.size, (6, 5))
        self.assertEqual(source_array_image.tile_size, (2, 2))
        self.assertEqual(source_array_image.num_tiles, (3, 3))

        self.assertEqual(source_array_image.get_tile(0, 0).tolist(),
                         [[24, 25],
                          [18, 19]])
        self.assertEqual(source_array_image.get_tile(1, 0).tolist(),
                         [[26, 27],
                          [20, 21]])
        self.assertEqual(source_array_image.get_tile(2, 0).tolist(),
                         [[28, 29],
                          [22, 23]])

        self.assertEqual(source_array_image.get_tile(0, 1).tolist(),
                         [[12, 13],
                          [6, 7]])
        self.assertEqual(source_array_image.get_tile(1, 1).tolist(),
                         [[14, 15],
                          [8, 9]])
        self.assertEqual(source_array_image.get_tile(2, 1).tolist(),
                         [[16, 17],
                          [10, 11]])

        tile_0_2 = source_array_image.get_tile(0, 2).tolist()
        self.assertEqual(tile_0_2[0], [0, 1])
        self.assertTrue(np.isnan(tile_0_2[1][0]))
        self.assertTrue(np.isnan(tile_0_2[1][1]))
        tile_1_2 = source_array_image.get_tile(1, 2).tolist()
        self.assertEqual(tile_1_2[0], [2, 3])
        self.assertTrue(tile_1_2[1][0])
        self.assertTrue(tile_1_2[1][1])
        tile_2_2 = source_array_image.get_tile(2, 2).tolist()
        self.assertEqual(tile_2_2[0], [4, 5])
        self.assertTrue(tile_2_2[1][0])
        self.assertTrue(tile_2_2[1][1])


class TrimTileTest(TestCase):
    # noinspection PyMethodMayBeStatic
    def test_trim_tile(self):
        a = np.arange(0, 6, dtype=np.float32)
        a.shape = 2, 3
        b = trim_tile(a, (3, 2))
        np.testing.assert_equal(b, np.array([[0., 1., 2.],
                                             [3., 4., 5.]]))
        b = trim_tile(a, (4, 2))
        np.testing.assert_equal(b, np.array([[0., 1., 2., np.nan],
                                             [3., 4., 5., np.nan]]))
        b = trim_tile(a, (3, 3))
        np.testing.assert_equal(b, np.array([[0., 1., 2.],
                                             [3., 4., 5.],
                                             [np.nan, np.nan, np.nan]]))
        b = trim_tile(a, (4, 3))
        np.testing.assert_equal(b, np.array([[0., 1., 2., np.nan],
                                             [3., 4., 5., np.nan],
                                             [np.nan, np.nan, np.nan, np.nan]]))


# test/util/test_tilegrid.py

import unittest
from unittest import TestCase

import pyproj

from xcube.util.tilegrid import GeographicTileGrid
from xcube.util.tilegrid import ImageTileGrid
from xcube.util.tilegrid import tile_grid_to_ol_xyz_source_options


class Crs84TileGridTest(TestCase):
    def test_it(self):
        tg = GeographicTileGrid()
        self.assertEqual((256, 256), tg.tile_size)
        self.assertEqual(None, tg.min_level)
        self.assertEqual(None, tg.max_level)
        self.assertEqual(pyproj.CRS.from_string('CRS84'), tg.crs)
        self.assertEqual((-180, -90, 180, 90), tg.extent)
        self.assertEqual((-180, 90), tg.origin)
        self.assertEqual(False, tg.is_j_axis_up)
        self.assertEqual((512, 256), tg.get_image_size(0))
        self.assertEqual((1024, 512), tg.get_image_size(1))
        self.assertEqual((512 * 2 ** 15, 256 * 2 ** 15), tg.get_image_size(15))
        self.assertEqual((2, 1), tg.get_num_tiles(0))
        self.assertEqual((4, 2), tg.get_num_tiles(1))
        self.assertEqual((2 ** 16, 2 ** 15), tg.get_num_tiles(15))
        self.assertEqual(180 / 256, tg.get_resolution(0))
        self.assertEqual(180 / 512, tg.get_resolution(1))
        self.assertEqual(180 / (256 * 2 ** 15), tg.get_resolution(15))


class ImageTileGridTest(TestCase):
    def test_it(self):
        tg = ImageTileGrid(image_size=(4000, 2000),
                           tile_size=(512, 512),
                           crs=pyproj.CRS.from_string('WGS84'),
                           xy_res=0.05,
                           xy_min=(0, 0),
                           is_j_axis_up=True)
        self.assertEqual((512, 512), tg.tile_size)
        self.assertEqual(None, tg.min_level)
        self.assertEqual(2, tg.max_level)
        self.assertEqual(pyproj.CRS.from_string('WGS84'), tg.crs)
        self.assertEqual((0, 0, 200.0, 100.0), tg.extent)
        self.assertEqual((0, 100.0), tg.origin)
        self.assertEqual(True, tg.is_j_axis_up)
        self.assertEqual((1000, 500), tg.get_image_size(0))
        self.assertEqual((2000, 1000), tg.get_image_size(1))
        self.assertEqual((4000, 2000), tg.get_image_size(2))
        self.assertEqual((2, 1), tg.get_num_tiles(0))
        self.assertEqual((4, 2), tg.get_num_tiles(1))
        self.assertEqual((8, 4), tg.get_num_tiles(2))
        self.assertEqual(0.2, tg.get_resolution(0))
        self.assertEqual(0.1, tg.get_resolution(1))
        self.assertEqual(0.05, tg.get_resolution(2))


class OpenLayersOptionsTest(unittest.TestCase):
    def test_geographic_only(self):
        tile_grid = ImageTileGrid(image_size=(7200, 3600),
                                  tile_size=(512, 512),
                                  crs='EPSG:32633',
                                  xy_res=180 / 3600,
                                  xy_min=(-180, -90),
                                  is_j_axis_up=True)
        self.assertIsNone(tile_grid_to_ol_xyz_source_options(
            tile_grid,
            'https://xcube.server/tile/{x}/{y}/{z}.png'
        ))

    def test_with_max_level(self):
        tile_grid = ImageTileGrid(image_size=(7200, 3600),
                                  tile_size=(512, 512),
                                  crs='EPSG:4326',
                                  xy_res=180. / 3600,
                                  xy_min=(-180., -90.),
                                  is_j_axis_up=True)
        self.assertEqual(
            {
                'url': 'https://xcube.server/tile/{x}/{y}/{z}.png',
                'projection': 'EPSG:4326',
                'tileGrid': {
                    'tileSize': [512, 512],
                    'sizes': [[2, 1], [4, 2], [8, 4], [15, 8]],
                    'resolutions': [0.4, 0.2, 0.1, 0.05],
                    'extent': [-180., -90., 180., 90.],
                    'origin': [-180., 90.],
                },
            },
            tile_grid_to_ol_xyz_source_options(
                tile_grid,
                'https://xcube.server/tile/{x}/{y}/{z}.png'
            )
        )

    def test_with_wo_max_level(self):
        tile_grid = GeographicTileGrid()
        self.assertEqual(
            {
                'url': 'https://xcube.server/tile/{x}/{y}/{z}.png',
                'projection': 'EPSG:4326',
                'tileSize': [256, 256],
                'maxResolution': 0.703125,
            },
            tile_grid_to_ol_xyz_source_options(
                tile_grid,
                'https://xcube.server/tile/{x}/{y}/{z}.png'
            )
        )


# test/util/test_types.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import unittest

from xcube.util.types import normalize_scalar_or_pair


class NormalizeNumberScalarOrPairTest(unittest.TestCase):

    def test_scalar_float(self):
        self.assertEqual((1.1, 1.1),
                         normalize_scalar_or_pair(1.1,
                                                  item_type=float))
        self.assertEqual((2.0, 2.0),
                         normalize_scalar_or_pair(2.0,
                                                  item_type=float))

    def test_scalar_int(self):
        self.assertEqual((1, 1), normalize_scalar_or_pair(1, item_type=int))
        self.assertEqual((1, 1), normalize_scalar_or_pair(1, item_type=int))
        self.assertEqual((2, 2), normalize_scalar_or_pair(2, item_type=int))

    def test_pair_float(self):
        self.assertEqual((1.1, 1.9),
                         normalize_scalar_or_pair((1.1, 1.9),
                                                  item_type=float))
        self.assertEqual((2.0, 1.9),
                         normalize_scalar_or_pair((2.0, 1.9),
                                                  item_type=float))

    def test_pair_int(self):
        self.assertEqual((1, 1),
                         normalize_scalar_or_pair((1, 1), item_type=int))
        self.assertEqual((2, 1),
                         normalize_scalar_or_pair((2, 1), item_type=int))

    def test_wrong_type(self):
        with self.assertRaises(ValueError) as ve:
            normalize_scalar_or_pair('xyz',
                                     item_type=int,
                                     name='notastring')
        self.assertEqual(
            "notastring must be a scalar or pair of <class 'int'>, was 'xyz'",
            f'{ve.exception}'
        )

    def test_wrong_length(self):
        with self.assertRaises(ValueError) as ve:
            normalize_scalar_or_pair((0, 1, 2))
        self.assertEqual(
            "Value must be a scalar or pair of scalars, was '(0, 1, 2)'",
            f'{ve.exception}'
        )

# test/util/test_undefined.py

import copy
from unittest import TestCase

from xcube.util.undefined import UNDEFINED


class UndefinedTest(TestCase):
    def test_it(self):
        self.assertIsNotNone(UNDEFINED)
        self.assertEqual(str(UNDEFINED), 'UNDEFINED')
        self.assertEqual(repr(UNDEFINED), 'UNDEFINED')
        undefined_copy = copy.deepcopy(UNDEFINED)
        self.assertEqual(UNDEFINED, undefined_copy)
        self.assertEqual(hash(UNDEFINED), hash(undefined_copy))


# test/util/test_versions.py

import unittest

import xcube.version as xcube_version
from xcube.util.versions import get_xcube_versions


class GetVersionsTest(unittest.TestCase):

    def test_it_contains_what_is_expected(self):
        versions = get_xcube_versions()
        self.assertIsInstance(versions, dict)
        self.assertTrue(all(isinstance(k, str) for k in versions.keys()))
        self.assertTrue(all(isinstance(v, str) for v in versions.values()))
        self.assertIn('xarray', versions)
        self.assertIn('dask', versions)
        self.assertIn('zarr', versions)
        self.assertIn('tornado', versions)
        self.assertIn('pandas', versions)
        self.assertIn('numpy', versions)
        self.assertIn('geopandas', versions)
        self.assertIn('xcube', versions)
        self.assertEqual(xcube_version, versions['xcube'])

    def test_it_is_cached(self):
        versions = get_xcube_versions()
        self.assertIs(versions, get_xcube_versions())


# setup.py

#!/usr/bin/env python3

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

from setuptools import setup, find_packages

requirements = [
    #
    # xcube requirements are given in file ./environment.yml.
    #
    # All packages here have been commented out,
    # because otherwise setuptools will install
    # additional pip packages although conda packages
    # with same name are already available
    # in the conda environment defined by file ./environment.yml.
    #
    # 'affine',
    # 'click',
    # 'cmocean',
    # 'dask',
    # 'fiona',
    # 'gdal',
    # 'matplotlib',
    # 'netcdf4',
    # 'numba',
    # 'numpy',
    # 'pandas',
    # 'pillow',
    # 'proj4',
    # 'pyyaml',
    # 'rasterio',
    # 's3fs',
    # 'setuptools',
    # 'shapely',
    # 'tornado',
    # 'xarray',
    # 'zarr',
]

packages = find_packages(exclude=["test", "test.*"])

# Same effect as "from cate import version", but avoids importing cate:
version = None
with open('xcube/version.py') as f:
    exec(f.read())

setup(
    name="xcube",
    version=version,
    description=('xcube is a Python package for generating and exploiting '
                 'data cubes powered by xarray, dask, and zarr.'),
    license='MIT',
    author='xcube Development Team',
    packages=packages,
    entry_points={
        'console_scripts': [
            # xcube's CLI
            'xcube = xcube.cli.main:main',
        ],
        'xcube_plugins': [
            # xcube's default extensions
            'xcube = xcube.plugin:init_plugin',
        ],
    },
    install_requires=requirements,
)


# test/webapi/_auth/__init__.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

# test/webapi/_auth/test_config.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest

import pytest

from xcube.server.api import ApiError
# noinspection PyProtectedMember
from xcube.webapi._auth.config import AuthConfig


class AuthContextTest(unittest.TestCase):
    def test_from_config_empty(self):
        config = AuthConfig.from_config({})
        self.assertEqual(None, config)

    def test_from_config_ok(self):
        config = AuthConfig.from_config({
            "Authentication": {
                "Authority": "https://auth.com",
                "Audience": "myapi",
                "IsRequired": True,
            }
        })
        self.assertEqual("https://auth.com", config.authority)
        self.assertEqual("myapi", config.audience)
        self.assertEqual(["RS256"], config.algorithms)
        self.assertEqual(True, config.is_required)
        self.assertEqual('https://auth.com/.well-known/jwks.json',
                         config.well_known_jwks)
        self.assertEqual('https://auth.com/.well-known/openid-configuration',
                         config.well_known_oid_config)

    def test_from_config_with_domain_ok(self):
        config = AuthConfig.from_config({
            "Authentication": {
                "Domain": "auth.com",
                "Audience": "myapi",
            }
        })
        self.assertEqual("https://auth.com", config.authority)
        self.assertEqual("myapi", config.audience)
        self.assertEqual(["RS256"], config.algorithms)
        self.assertEqual(False, config.is_required)

    def test_from_config_fails(self):
        with pytest.raises(
                ApiError.InvalidServerConfig,
                match='HTTP status 580:'
                      ' Missing key "Authority" in section'
                      ' "Authentication"'
        ):
            AuthConfig.from_config({
                "Authentication": {
                }
            })

        with pytest.raises(
                ApiError.InvalidServerConfig,
                match='HTTP status 580:'
                      ' Missing key "Audience" in section'
                      ' "Authentication"'
        ):
            AuthConfig.from_config({
                "Authentication": {
                    "Authority": "https://auth.com"
                }
            })

        with pytest.raises(
                ApiError.InvalidServerConfig,
                match='HTTP status 580:'
                      ' Value for key "Algorithms" in section'
                      ' "Authentication" must not be empty'
        ):
            AuthConfig.from_config({
                "Authentication": {
                    "Authority": "https://auth.com",
                    "Audience": "myapi",
                    "Algorithms": [],
                }
            })


# test/webapi/_auth/test_context.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import json
import os
import types
import unittest

import requests

from test.webapi.helpers import get_api_ctx
from xcube.server.api import ApiError
from xcube.server.api import ServerConfig
# noinspection PyProtectedMember
# noinspection PyProtectedMember
from xcube.webapi._auth.config import AuthConfig
from xcube.webapi._auth.context import AuthContext

XCUBE_TEST_CLIENT_ID = os.environ.get('XCUBE_TEST_CLIENT_ID')
XCUBE_TEST_CLIENT_SECRET = os.environ.get('XCUBE_TEST_CLIENT_SECRET')


def get_auth_ctx(config: ServerConfig) -> AuthContext:
    return get_api_ctx("auth", AuthContext, config)


class AuthMixinPropsTest2(unittest.TestCase):
    def test_auth_config(self):
        auth_ctx = get_auth_ctx({
            "Authentication": {
                "Authority": "https://auth.com",
                "Audience": "myapi"
            }
        })
        auth_config = auth_ctx.auth_config
        self.assertIsInstance(auth_config, AuthConfig)
        # Assert that it is a cached property
        self.assertIs(auth_config, auth_ctx.auth_config)

    def test_granted_scopes(self):
        # Use "permissions" claim
        self.assert_granted_scopes(
            {
                "preferred_username": "bibbi",
                "permissions": [
                    "read:dataset:*~$preferred_username/*",
                    "read:variable:*"
                ]
            },
            {
                "read:dataset:*~bibbi/*",
                "read:variable:*"
            }
        )
        # Use "scope" claim
        self.assert_granted_scopes(
            {
                "preferred_username": "bibbi",
                "scope":
                    "read:dataset:*~$preferred_username/* read:variable:*"
            },
            {
                "read:dataset:*~bibbi/*",
                "read:variable:*"
            }
        )
        # Special case where "preferred_username" claim
        # is named just "username"
        self.assert_granted_scopes(
            {
                "preferred_username": "bibbi",
                "permissions": [
                    "read:dataset:*~${username}/*",
                    "read:variable:*"
                ]
            },
            {
                "read:dataset:*~bibbi/*",
                "read:variable:*"
            }
        )

    def assert_granted_scopes(self, id_token, expected_permissions):
        auth_ctx = get_auth_ctx({
            "Authentication": {
                "Authority": "https://auth.com",
                "Audience": "myapi"
            }
        })

        # noinspection PyUnusedLocal,PyShadowingNames
        def _get_id_token(self, headers, require_auth=False):
            return id_token

        auth_ctx.get_id_token = types.MethodType(_get_id_token, auth_ctx)

        self.assertEqual(expected_permissions,
                         auth_ctx.granted_scopes({}))


class AuthContextIdTokenTest(unittest.TestCase):
    # TODO (forman): setup Keycloak xcube test account
    @unittest.skipUnless(
        XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET,
        'XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET must be set'
    )
    def test_ok(self):
        auth_ctx = get_auth_ctx({
            "Authentication": {
                "Domain": "xcube-dev.eu.auth0.com",
                "Audience": "https://xcube-dev/api/",
            }
        })

        access_token = self._fetch_access_token()

        id_token = auth_ctx.get_id_token(
            {'Authorization': f'Bearer {access_token}'}
        )
        self.assertEqual('https://xcube-dev.eu.auth0.com/',
                         id_token.get('iss'))
        self.assertEqual('https://xcube-dev/api/',
                         id_token.get('aud'))
        self.assertEqual('w2NoDEryIpNRtAQVH1ToU6XTurE549FW',
                         id_token.get('azp'))
        self.assertEqual('client-credentials',
                         id_token.get('gty'))
        self.assertEqual('w2NoDEryIpNRtAQVH1ToU6XTurE549FW@clients',
                         id_token.get('sub'))
        self.assertEqual(['read:datasets'],
                         id_token.get('permissions'))
        exp = id_token.get('exp')
        iat = id_token.get('iat')
        self.assertIsInstance(exp, int)
        self.assertIsInstance(iat, int)
        self.assertEqual(86400, exp - iat)

    def _fetch_access_token(self):
        response = requests.post(
            'https://xcube-dev.eu.auth0.com/oauth/token',
            headers={'Content-Type': 'application/json'},
            json={
                "client_id": os.environ.get('XCUBE_TEST_CLIENT_ID'),
                "client_secret": os.environ.get('XCUBE_TEST_CLIENT_SECRET'),
                "audience": "https://xcube-dev/api/",
                "grant_type": "client_credentials"
            }
        )
        token_data = json.loads(response.content)
        if 'access_token' not in token_data:
            self.fail('failed to fetch access token for testing')
        access_token = token_data['access_token']
        return access_token

    def test_missing_auth_config(self):
        auth_ctx = get_auth_ctx({})
        self.assertIsNone(
            auth_ctx.get_id_token(
                {'Authorization': f'Bearer my_t0k3n'},
                require_auth=True
            )
        )

    def test_missing_access_token(self):
        auth_ctx = get_auth_ctx({
            "Authentication": {
                "Domain": "xcube-dev.eu.auth0.com",
                "Audience": "https://xcube-dev/api/",
            }
        })
        self.assertEqual(None, auth_ctx.get_id_token({}))

    # TODO (forman): setup Keycloak xcube test account
    @unittest.skipUnless(
        XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET,
        'XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET must be set'
    )
    def test_expired_access_token(self):
        auth_ctx = get_auth_ctx({
            "Authentication": {
                "Domain": "xcube-dev.eu.auth0.com",
                "Audience": "https://xcube-dev/api/",
            }
        })

        expired_token = \
            "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6" \
            "Ik5VSkdSVUV5UWpNeE16UTVRVGMzTnpRM05URkNRa1ky" \
            "TURGRE5qQkJSak5ETlRBeVFrWXdOUSJ9.eyJpc3MiOiJ" \
            "odHRwczovL3hjdWJlLWRldi5ldS5hdXRoMC5jb20vIiw" \
            "ic3ViIjoidzJOb0RFcnlJcE5SdEFRVkgxVG9VNlhUdXJ" \
            "FNTQ5RldAY2xpZW50cyIsImF1ZCI6Imh0dHBzOi8veGN" \
            "1YmUtZGV2L2FwaS8iLCJpYXQiOjE1Nzk0NTUxMDMsImV" \
            "4cCI6MTU3OTU0MTUwMywiYXpwIjoidzJOb0RFcnlJcE5" \
            "SdEFRVkgxVG9VNlhUdXJFNTQ5RlciLCJndHkiOiJjbGl" \
            "lbnQtY3JlZGVudGlhbHMiLCJwZXJtaXNzaW9ucyI6W11" \
            "9.NtGuVp0yY8foR_eFiEmH0eXvCx85cvb5b1cPWCKs6L" \
            "CNniUJJ3VnX5Fek08puW7Jidy-tj1UTzyG569dSHGDu3" \
            "10Mf7xpQ9gyfZCWcaohERxsv9MrxHziqfGVnxv051rOB" \
            "_c-fYyymrFnlsIqWeahcS7znvPoMovPO7E8MxVaIqxd_" \
            "S4zKVlH025F-bDvytuwXD-rFmYVElCg7u2uOZKqjpF3l" \
            "ZCWc50_F1jSGcEQZv4daQJY-3lfU6TnEQAuGWlOVRrCN" \
            "u05nlUBFPz6G82tB_nsP1uTa8uElOzoalVttXufLIeU0" \
            "FL8Sv-lC6wUJTZAFpykLNmpA-vhkSeTqMv4g"
        with self.assertRaises(ApiError.Unauthorized) as cm:
            auth_ctx.get_id_token(
                {'Authorization': f'Bearer {expired_token}'}
            )
        self.assertEqual('HTTP status 401: Token is expired.',
                         f'{cm.exception}')


class AuthContextGetAccessTokenTest(unittest.TestCase):

    def test_ok(self):
        headers = {'Authorization': 'Bearer my_t0k3n'}
        self.assertEqual('my_t0k3n', AuthContext.get_access_token(headers))

    def test_ok_no_auth(self):
        self.assertEqual(None, AuthContext.get_access_token({}))

    def test_not_ok(self):
        with self.assertRaises(ApiError.Unauthorized) as cm:
            AuthContext.get_access_token({}, require_auth=True)
        self.assertEqual('HTTP status 401: Authorization header is expected.',
                         f'{cm.exception}')

        with self.assertRaises(ApiError.BadRequest) as cm:
            AuthContext.get_access_token({'Authorization': 'Beerer my_t0k3n'})
        self.assertEqual('HTTP status 400: Invalid header.'
                         ' Authorization header must start with "Bearer".',
                         f'{cm.exception}')

        with self.assertRaises(ApiError.BadRequest) as cm:
            AuthContext.get_access_token({'Authorization': 'Bearer'})
        self.assertEqual('HTTP status 400: Invalid header.'
                         ' Bearer token not found.',
                         f'{cm.exception}')

        with self.assertRaises(ApiError.BadRequest) as cm:
            AuthContext.get_access_token({'Authorization': 'Bearer my t0k3n'})
        self.assertEqual('HTTP status 400: Invalid header.'
                         ' Authorization header must be Bearer token.',
                         f'{cm.exception}')


# test/webapi/controllers/test_catalogue.py

import os.path
import unittest
from typing import Any, Optional

from test.webapi.helpers import new_test_service_context
from xcube.core.new import new_cube
from xcube.webapi.context import ServiceContext
from xcube.webapi.controllers.catalogue import get_color_bars
from xcube.webapi.controllers.catalogue import get_dataset
from xcube.webapi.controllers.catalogue import get_datasets
from xcube.webapi.controllers.catalogue import get_time_chunk_size
from xcube.webapi.errors import ServiceBadRequestError, ServiceAuthError


def new_ctx():
    data_store_root = os.path.join(
        os.path.dirname(__file__),
        '..', '..', '..', 'examples', 'serve', 'demo'
    )
    config = dict(
        Authentication=dict(
            Domain='xcube-dev.eu.auth0.com',
            Audience='https://xcube-dev/api/',
        ),
        AccessControl=dict(
            RequiredScopes=[
                'read:dataset:{Dataset}',
                'read:variable:{Variable}'
            ]
        ),
        DataStores=[
            dict(
                Identifier='local',
                StoreId='file',
                StoreParams=dict(
                    root=data_store_root
                ),
                Datasets=[
                    # "base" will only appear for unauthorized clients
                    dict(
                        Identifier='local_base_id',
                        Title='A local base dataset',
                        Path='cube-1-250-250.zarr',
                        TimeSeriesDataset='local~cube-5-100-200.zarr',
                        AccessControl=dict(
                            IsSubstitute=True
                        )
                    ),
                    # "time_series" will not appear at all,
                    # because it is a "hidden" resource
                    dict(
                        Identifier='time_chunked',
                        Path='cube-5-100-200.zarr',
                        Hidden=True,
                    ),
                ]
            ),
            dict(
                Identifier='remote',
                StoreId='s3',
                StoreParams=dict(
                    root='xcube-examples',
                    storage_options=dict(anon=True),
                ),
                Datasets=[
                    dict(
                        Title='A remote base dataset',
                        Path='OLCI-SNS-RAW-CUBE-2.zarr'
                    ),
                ]
            )
        ],
        Datasets=[
            # "local" will only appear for unauthorized clients
            dict(
                Identifier='local_base_1w',
                FileSystem='memory',
                Path='examples/serve/demo/resample_in_time.py',
                Function='compute_dataset',
                InputDatasets=['local_base_id'],
                InputParameters=dict(
                    period='1W',
                    incl_stdev=True
                ),
                AccessControl=dict(
                    IsSubstitute=True
                )
            ),
            dict(
                Identifier='remote_base_1w',
                FileSystem='memory',
                Path='examples/serve/demo/resample_in_time.py',
                Function='compute_dataset',
                InputDatasets=['remote~OLCI-SNS-RAW-CUBE-2.zarr'],
                InputParameters=dict(
                    period='1W',
                    incl_stdev=True
                ),
            ),
        ],
    )
    return ServiceContext(config=config)


class CatalogueControllerTest(unittest.TestCase):
    def test_unauthorized_access(self):
        ctx = new_ctx()
        granted_scopes = None
        response = get_datasets(ctx, granted_scopes=granted_scopes)
        datasets = self._assert_datasets(response)
        datasets_dict = {ds['id']: ds for ds in datasets}
        self.assertEqual(
            {
                'local_base_id',
                'local_base_1w',
                # Not selected, because they require authorisation
                # 'remote~OLCI-SNS-RAW-CUBE-2.zarr',
                # 'remote_base_1w',
            },
            set(datasets_dict)
        )
        dataset_titles_dict = {ds['title']: ds for ds in datasets}
        self.assertEqual(
            {
                'local_base_1w',
                'A local base dataset',
                # Not selected, because they require authorisation
                # 'remote_base_1w',
                # 'A remote base dataset',
            },
            set(dataset_titles_dict)
        )
        dataset = get_dataset(ctx, 'local_base_id')
        self.assertIn('variables', dataset)
        var_dict = {v['name']: v for v in dataset['variables']}
        self.assertEqual({'c2rcc_flags',
                          'conc_tsm',
                          'kd489',
                          'conc_chl',
                          'quality_flags'},
                         set(var_dict.keys()))

        dataset = get_dataset(ctx, 'local_base_1w')
        self.assertIn('variables', dataset)
        var_dict = {v['name']: v for v in dataset['variables']}
        self.assertEqual({'c2rcc_flags',
                          'c2rcc_flags_stdev',
                          'conc_chl',
                          'conc_chl_stdev',
                          'conc_tsm',
                          'conc_tsm_stdev',
                          'kd489',
                          'kd489_stdev',
                          'quality_flags',
                          'quality_flags_stdev'},
                         set(var_dict.keys()))

        with self.assertRaises(ServiceAuthError) as cm:
            get_dataset(ctx, 'remote_base_1w')
        self.assertEqual(
            'HTTP 401: Missing permission'
            ' (Missing permission'
            ' read:dataset:remote_base_1w)',
            f'{cm.exception}'
        )

        with self.assertRaises(ServiceAuthError) as cm:
            get_dataset(ctx, 'remote~OLCI-SNS-RAW-CUBE-2.zarr')
        self.assertEqual(
            'HTTP 401: Missing permission'
            ' (Missing permission'
            ' read:dataset:remote~OLCI-SNS-RAW-CUBE-2.zarr)',
            f'{cm.exception}'
        )

    def test_authorized_access_with_joker_scopes(self):
        ctx = new_ctx()
        granted_scopes = {
            'read:dataset:*',
            'read:variable:*'
        }
        response = get_datasets(ctx, granted_scopes=granted_scopes)
        datasets = self._assert_datasets(response)
        dataset_ids_dict = {ds['id']: ds for ds in datasets}
        self.assertEqual(
            {
                'local_base_1w',
                'local_base_id',
                'remote_base_1w',
                'remote~OLCI-SNS-RAW-CUBE-2.zarr',
            },
            set(dataset_ids_dict)
        )
        dataset_titles_dict = {ds['title']: ds for ds in datasets}
        self.assertEqual(
            {
                'local_base_1w',
                'A local base dataset',
                'remote_base_1w',
                'A remote base dataset',
            },
            set(dataset_titles_dict)
        )

    def test_authorized_access_with_specific_scopes(self):
        ctx = new_ctx()
        granted_scopes = {
            'read:dataset:remote*',
            'read:variable:*'
        }
        response = get_datasets(ctx, granted_scopes=granted_scopes)
        datasets = self._assert_datasets(response)
        dataset_ids_dict = {ds['id']: ds for ds in datasets}
        self.assertEqual(
            {
                # Not selected, because they are substitutes
                # 'local_base_1w',
                # 'local_base_id',
                'remote_base_1w',
                'remote~OLCI-SNS-RAW-CUBE-2.zarr',
            },
            set(dataset_ids_dict)
        )
        dataset_titles_dict = {ds['title']: ds for ds in datasets}
        self.assertEqual(
            {
                # Not selected, because they are substitutes
                # 'local_base_1w',
                # 'A local base dataset',
                'remote_base_1w',
                'A remote base dataset',
            },
            set(dataset_titles_dict)
        )

    def test_datasets(self):
        ctx = new_test_service_context()
        response = get_datasets(ctx)
        datasets = self._assert_datasets(response, expected_count=2)
        for dataset in datasets:
            self.assertIsInstance(dataset, dict)
            self.assertIn("id", dataset)
            self.assertIn("title", dataset)
            self.assertNotIn("variables", dataset)
            self.assertNotIn("dimensions", dataset)
            self.assertNotIn("rgbSchema", dataset)

    def test_dataset_with_details(self):
        ctx = new_test_service_context()
        response = get_datasets(ctx, details=True, base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=2)

        demo_dataset = None
        demo_1w_dataset = None
        for dataset in datasets:
            self.assertIsInstance(dataset, dict)
            self.assertIsInstance(dataset.get("id"), str)
            self.assertIsInstance(dataset.get("title"), str)
            self.assertIsInstance(dataset.get("attributions"), (tuple, list))
            self.assertIsInstance(dataset.get("variables"), (tuple, list))
            self.assertIsInstance(dataset.get("dimensions"), (tuple, list))
            self.assertIsInstance(dataset.get("bbox"), (tuple, list))
            self.assertIsInstance(dataset.get("geometry"), dict)
            self.assertIsInstance(dataset.get("spatialRef"), str)
            self.assertNotIn("rgbSchema", dataset)
            if dataset["id"] == "demo":
                demo_dataset = dataset
            if dataset["id"] == "demo-1w":
                demo_1w_dataset = dataset

        self.assertIsNotNone(demo_dataset)
        self.assertIsNotNone(demo_1w_dataset)
        self.assertEqual(['© by EU H2020 CyanoAlert project'],
                         demo_dataset['attributions'])
        self.assertEqual(["© by Brockmann Consult GmbH 2020, "
                          "contains modified Copernicus Data 2019, "
                          "processed by ESA"],
                         demo_1w_dataset['attributions'])

    def test_dataset_with_details_and_rgb_schema(self):
        ctx = new_test_service_context('config-rgb.yml')
        response = get_datasets(ctx, details=True, base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=1)
        dataset = datasets[0]
        self.assertIsInstance(dataset, dict)
        self.assertEqual(
            {
                'varNames': ['conc_chl', 'conc_tsm', 'kd489'],
                'normRanges': [(0.0, 24.0), (0.0, 100.0),
                               (0.0, 6.0)],
                'tileUrl': 'http://test/datasets/demo-rgb/vars/rgb/tiles2/{z}/{y}/{x}',
                'tileLevelMin': 7,
                'tileLevelMax': 9,
            },
            dataset.get("rgbSchema")
        )
        response = get_datasets(ctx, details=True, client='ol4',
                                base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=1)
        dataset = datasets[0]
        self.assertIsInstance(dataset, dict)
        self.assertEqual(
            {
                'varNames': ['conc_chl', 'conc_tsm', 'kd489'],
                'normRanges': [(0.0, 24.0), (0.0, 100.0),
                               (0.0, 6.0)],
                'tileUrl': 'http://test/datasets/demo-rgb/vars/rgb/tiles2/{z}/{y}/{x}',
                'tileLevelMin': 7,
                'tileLevelMax': 9,
                'tileSourceOptions': {
                    'url': 'http://test/datasets/demo-rgb/vars/rgb/tiles/{z}/{x}/{y}.png',
                    'projection': 'EPSG:4326',
                    'tileGrid': {'extent': [0, 50, 5, 52.5],
                                 'origin': [0, 52.5],
                                 'resolutions': [0.01, 0.005,
                                                 0.0025],
                                 'sizes': [[2, 1], [4, 2], [8, 4]],
                                 'tileSize': [250, 250]}
                }
            },
            dataset.get("rgbSchema")
        )

    def test_dataset_with_point(self):
        ctx = new_test_service_context()

        response = get_datasets(ctx, point=(1.7, 51.2),
                                base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=2)
        dataset = datasets[0]
        self.assertIsInstance(dataset, dict)
        self.assertIn("id", dataset)
        self.assertIn("title", dataset)
        self.assertNotIn("variables", dataset)
        self.assertNotIn("dimensions", dataset)

        response = get_datasets(ctx, point=(1.7, 58.0),
                                base_url="http://test")
        self._assert_datasets(response, expected_count=0)

    def test_dataset_with_point_and_details(self):
        ctx = new_test_service_context()

        response = get_datasets(ctx, point=(1.7, 51.2), details=True,
                                base_url="http://test")
        datasets = self._assert_datasets(response, expected_count=2)
        dataset = datasets[0]
        self.assertIsInstance(dataset, dict)
        self.assertIn("id", dataset)
        self.assertIn("title", dataset)
        self.assertIn("variables", dataset)
        self.assertIn("dimensions", dataset)

        response = get_datasets(ctx, point=(1.7, 58.0), details=True,
                                base_url="http://test")
        self._assert_datasets(response, 0)

    def test_get_colorbars(self):
        ctx = ServiceContext()

        response = get_color_bars(ctx, 'application/json')
        self.assertIsInstance(response, str)
        self.assertTrue(len(response) > 40)
        self.assertEqual('[\n  [\n    "Perceptually Uniform Sequenti',
                         response[0:40])

        response = get_color_bars(ctx, 'text/html')
        self.assertIsInstance(response, str)
        self.assertTrue(len(response) > 40)
        self.assertEqual('<!DOCTYPE html>\n<html lang="en">\n<head><',
                         response[0:40])

        with self.assertRaises(ServiceBadRequestError) as cm:
            get_color_bars(ctx, 'text/xml')
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual("Format 'text/xml' not supported for color bars",
                         cm.exception.reason)

    def _assert_datasets(self,
                         response: Any,
                         expected_count: Optional[int] = None):
        self.assertIsInstance(response, dict)
        datasets = response.get("datasets")
        self.assertIsInstance(datasets, list)
        if expected_count is not None:
            self.assertEqual(expected_count, len(datasets))
        return datasets


class TimeChunkSizeTest(unittest.TestCase):

    @staticmethod
    def _get_cube(time_chunk_size: int = None):
        ts_ds = new_cube(time_periods=10, variables=dict(CHL=10.2))
        if time_chunk_size:
            ts_ds = ts_ds.assign(
                CHL=ts_ds.CHL.chunk(dict(time=time_chunk_size))
            )
        return ts_ds

    def test_get_time_chunk_size_is_ok(self):
        ts_ds = self._get_cube(time_chunk_size=1)
        self.assertEqual(1, get_time_chunk_size(ts_ds, 'CHL', 'ds.zarr'))

        ts_ds = self._get_cube(time_chunk_size=3)
        self.assertEqual(3, get_time_chunk_size(ts_ds, 'CHL', 'ds.zarr'))

        ts_ds = self._get_cube(time_chunk_size=5)
        self.assertEqual(5, get_time_chunk_size(ts_ds, 'CHL', 'ds.zarr'))

        ts_ds = self._get_cube(time_chunk_size=10)
        self.assertEqual(10, get_time_chunk_size(ts_ds, 'CHL', 'ds.zarr'))

    def test_get_time_chunk_size_fails(self):
        # TS dataset not given
        self.assertEqual(None, get_time_chunk_size(None, 'CHL', 'ds.zarr'))
        # Variable not found
        ts_ds = self._get_cube(time_chunk_size=5)
        self.assertEqual(None, get_time_chunk_size(ts_ds, 'MCI', 'ds.zarr'))
        # Time is not chunked
        ts_ds = self._get_cube(time_chunk_size=None)
        self.assertEqual(None, get_time_chunk_size(ts_ds, 'CHL', 'ds.zarr'))
        # Variable has no dimension "time"
        ts_ds = self._get_cube(time_chunk_size=5)
        ts_ds['CHL0'] = ts_ds.CHL.isel(time=0)
        self.assertEqual(None, get_time_chunk_size(ts_ds, 'CHL0', 'ds.zarr'))


# test/webapi/controllers/test_places.py

import unittest

from test.webapi.helpers import new_test_service_context
from xcube.webapi.controllers.places import find_places, find_dataset_places
from xcube.webapi.errors import ServiceBadRequestError


class FeaturesControllerTest(unittest.TestCase):

    def test_find_places_all(self):
        ctx = new_test_service_context()
        places = find_places(ctx, "all", "http://localhost:9090")
        self._assertPlaceGroup(places, 6, {'0', '1', '2', '3', '4', '5'})

    def test_find_places_by_box(self):
        ctx = new_test_service_context()
        places = find_places(ctx, "all", "http://localhost:9090", box_coords="-1,49,2,55")
        self._assertPlaceGroup(places, 2, {'0', '3'})

        with self.assertRaises(ServiceBadRequestError) as cm:
            find_places(ctx, "all", "http://localhost:9090", box_coords="-1,49,55")
        self.assertEqual("HTTP 400: Received invalid bounding box geometry", f"{cm.exception}")

        with self.assertRaises(ServiceBadRequestError) as cm:
            find_places(ctx, "all", "http://localhost:9090", box_coords="-1,49,x,55")
        self.assertEqual("HTTP 400: Received invalid bounding box geometry", f"{cm.exception}")

    def test_find_places_by_wkt(self):
        ctx = new_test_service_context()
        places = find_places(ctx, "all", "http://localhost:9090", geom_wkt="POLYGON ((2 49, 2 55, -1 55, -1 49, 2 49))")
        self._assertPlaceGroup(places, 2, {'0', '3'})

        with self.assertRaises(ServiceBadRequestError) as cm:
            find_places(ctx, "all", "http://localhost:9090", geom_wkt="POLYGLON ((2 49, 2 55, -1 55, -1 49, 2 49))")
        self.assertEqual("HTTP 400: Received invalid geometry WKT", f"{cm.exception}")

    def test_find_places_by_geojson(self):
        ctx = new_test_service_context()

        geojson_obj = {'type': 'Polygon',
                       'coordinates': (((2.0, 49.0), (2.0, 55.0), (-1.0, 55.0), (-1.0, 49.0), (2.0, 49.0)),)}
        places = find_places(ctx, "all", "http://localhost:9090", geojson_obj=geojson_obj)
        self._assertPlaceGroup(places, 2, {'0', '3'})

        geojson_obj = {'type': 'Feature', 'properties': {}, 'geometry': geojson_obj}
        places = find_places(ctx, "all", "http://localhost:9090", geojson_obj=geojson_obj)
        self._assertPlaceGroup(places, 2, {'0', '3'})

        geojson_obj = {'type': 'FeatureCollection', 'features': [geojson_obj]}
        places = find_places(ctx, "all", "http://localhost:9090", geojson_obj=geojson_obj)
        self._assertPlaceGroup(places, 2, {'0', '3'})

        with self.assertRaises(ServiceBadRequestError) as cm:
            geojson_obj = {'type': 'FeatureCollection', 'features': []}
            find_places(ctx, "all", "http://localhost:9090", geojson_obj=geojson_obj)
        self.assertEqual("HTTP 400: Received invalid GeoJSON object", f"{cm.exception}")

    def test_find_dataset_features(self):
        ctx = new_test_service_context()
        places = find_dataset_places(ctx, "all", "demo", "http://localhost:8080")
        self._assertPlaceGroup(places, 3, {'0', '1', '2'})

    def _assertPlaceGroup(self, feature_collection, expected_count, expected_ids):
        self.assertIsInstance(feature_collection, dict)
        self.assertIn("type", feature_collection)
        self.assertEqual("FeatureCollection", feature_collection["type"])
        self.assertIn("features", feature_collection)
        features = feature_collection["features"]
        self.assertIsInstance(features, list)
        self.assertEqual(expected_count, len(features))
        actual_ids = {f["id"] for f in features if "id" in f}
        self.assertEqual(expected_ids, actual_ids)


# test/webapi/controllers/test_tiles.py

import unittest

from test.webapi.helpers import new_test_service_context, RequestParamsMock
from xcube.webapi.controllers.tiles import get_dataset_tile, get_dataset_tile_grid, \
    get_legend
from xcube.webapi.errors import ServiceBadRequestError, ServiceResourceNotFoundError


class TilesControllerTest(unittest.TestCase):

    def test_get_dataset_tile(self):
        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock())
        self.assertIsInstance(tile, bytes)

        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '-20', '0', '0', RequestParamsMock())
        self.assertIsInstance(tile, bytes)

    def test_get_dataset_tile_invalid_dataset(self):
        ctx = new_test_service_context()
        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            get_dataset_tile(ctx, 'demo-rgb', 'conc_tsm', '0', '0', '0', RequestParamsMock())
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Dataset "demo-rgb" not found', f'{cm.exception.reason}')

    def test_get_dataset_tile_invalid_variable(self):
        ctx = new_test_service_context()
        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            get_dataset_tile(ctx, 'demo', 'conc_tdi', '0', '0', '0', RequestParamsMock())
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Variable "conc_tdi" not found in dataset "demo"', f'{cm.exception.reason}')

    def test_get_dataset_tile_with_all_params(self):
        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='current', cbar='plasma',
                                                                                          vmin='0.1', vmax='0.3'))
        self.assertIsInstance(tile, bytes)

    def test_get_dataset_tile_with_time_dim(self):
        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='2017-01-26'))
        self.assertIsInstance(tile, bytes)

        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='2017-01-26/2017-01-27'))
        self.assertIsInstance(tile, bytes)

        ctx = new_test_service_context()
        tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='current'))
        self.assertIsInstance(tile, bytes)

    def test_get_dataset_tile_with_invalid_time_dim(self):
        ctx = new_test_service_context()
        with self.assertRaises(ServiceBadRequestError) as cm:
            get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='Gnaaark!'))
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual("'Gnaaark!' is not a valid value for dimension 'time'",
                         cm.exception.reason)

    def test_get_dataset_rgb_tile(self):
        ctx = new_test_service_context('config-rgb.yml')
        tile = get_dataset_tile(ctx, 'demo-rgb', 'rgb', '0', '0', '0', RequestParamsMock())
        self.assertIsInstance(tile, bytes)

    def test_get_dataset_rgb_tile_invalid_b(self):
        ctx = new_test_service_context('config-rgb.yml')
        with self.assertRaises(ServiceBadRequestError) as cm:
            get_dataset_tile(ctx, 'demo-rgb', 'rgb', '0', '0', '0', RequestParamsMock(b='refl_3'))
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual("Variable 'refl_3' not found in dataset 'demo-rgb'",
                         cm.exception.reason)

    def test_get_dataset_rgb_tile_no_vars(self):
        ctx = new_test_service_context()
        with self.assertRaises(ServiceBadRequestError) as cm:
            get_dataset_tile(ctx, 'demo', 'rgb', '0', '0', '0', RequestParamsMock())
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual("No variable in dataset 'demo' specified for RGB",
                         cm.exception.reason)

    def test_get_dataset_tile_grid(self):
        self.maxDiff = None

        ctx = new_test_service_context()
        tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol4', 'http://bibo')
        self.assertEqual(
            {
                'projection': 'EPSG:4326',
                'tileGrid': {'extent': [0, 50, 5, 52.5],
                             'origin': [0, 52.5],
                             'resolutions': [0.01, 0.005, 0.0025],
                             'sizes': [[2, 1], [4, 2], [8, 4]],
                             'tileSize': [250, 250]},
                'url': 'http://bibo/datasets/demo/vars/conc_chl/tiles2/{z}/{y}/{x}'
            },
            tile_grid)

        with self.assertRaises(NotImplementedError):
            get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'cesium', 'http://bibo')

        with self.assertRaises(ServiceBadRequestError) as cm:
            get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol2.json', 'http://bibo')
        self.assertEqual(400, cm.exception.status_code)
        self.assertEqual('Unknown tile client "ol2.json"', cm.exception.reason)

    def test_get_dataset_tile_grid_with_prefix(self):
        self.maxDiff = None

        ctx = new_test_service_context(prefix='api/v1')

        tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol4', 'http://bibo')
        self.assertEqual(
            {
                'projection': 'EPSG:4326',
                'tileGrid': {'extent': [0, 50, 5.0, 52.5],
                             'origin': [0, 52.5],
                             'resolutions': [0.01, 0.005, 0.0025],
                             'sizes': [[2, 1], [4, 2], [8, 4]],
                             'tileSize': [250, 250]},
                'url': 'http://bibo/api/v1/datasets/demo/vars/conc_chl/tiles2/{z}/{y}/{x}'
            },
            tile_grid)

    def test_get_legend(self):
        ctx = new_test_service_context()
        image = get_legend(ctx, 'demo', 'conc_chl', RequestParamsMock())
        self.assertEqual("<class 'bytes'>", str(type(image)))

        # This is fine, because we fall back to "viridis".
        image = get_legend(ctx, 'demo', 'conc_chl', RequestParamsMock(cbar='sun-shine'))
        self.assertEqual("<class 'bytes'>", str(type(image)))

        with self.assertRaises(ServiceBadRequestError) as cm:
            get_legend(ctx, 'demo', 'conc_chl', RequestParamsMock(vmin='sun-shine'))
        self.assertEqual("""Parameter "vmin" must be a number, but was 'sun-shine'""", cm.exception.reason)

        with self.assertRaises(ServiceBadRequestError) as cm:
            get_legend(ctx, 'demo', 'conc_chl', RequestParamsMock(width='sun-shine'))
        self.assertEqual("""Parameter "width" must be an integer, but was 'sun-shine'""", cm.exception.reason)

    @property
    def base_url(self):
        return f'http://bibo'


# test/webapi/controllers/test_timeseries.py

import json
import os
import unittest

import numpy as np
import pandas as pd
import xarray as xr

from test.mixins import AlmostEqualDeepMixin
from test.webapi.helpers import new_test_service_context
from xcube.webapi.context import ServiceContext
from xcube.webapi.controllers.timeseries import get_time_series, _collect_timeseries_result


class TimeSeriesControllerTest(unittest.TestCase, AlmostEqualDeepMixin):

    def setUp(self) -> None:
        self.maxDiff = None

    def test_get_time_series_for_point(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Point", coordinates=[2.1, 51.4]),
                                        start_date=np.datetime64('2017-01-15'),
                                        end_date=np.datetime64('2017-01-29'))
        expected_result = [{'mean': 3.534773588180542, 'time': '2017-01-16T10:09:22Z'},
                           {'mean': None, 'time': '2017-01-25T09:35:51Z'},
                           {'mean': None, 'time': '2017-01-26T10:50:17Z'},
                           {'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_out_of_bounds(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Point", coordinates=[-150.0, -30.0]))
        expected_result = []
        self.assertEqual(expected_result, actual_result)

    def test_get_time_series_for_point_one_valid(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Point", coordinates=[2.1, 51.4]),
                                        agg_methods=['mean', 'count'],
                                        start_date=np.datetime64('2017-01-15'),
                                        end_date=np.datetime64('2017-01-29'),
                                        max_valids=1)
        expected_result = [{'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_only_valids(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Point", coordinates=[2.1, 51.4]),
                                        start_date=np.datetime64('2017-01-15'),
                                        end_date=np.datetime64('2017-01-29'),
                                        max_valids=-1)
        expected_result = [{'mean': 3.534773588180542, 'time': '2017-01-16T10:09:22Z'},
                           {'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Polygon", coordinates=[[
                                            [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                        ]]),
                                        agg_methods=['mean', 'count'])
        expected_result = [{'count': 122392,
                            'count_tot': 159600,
                            'mean': 56.12519223634024,
                            'time': '2017-01-16T10:09:22Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'mean': None,
                            'time': '2017-01-25T09:35:51Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'mean': None,
                            'time': '2017-01-26T10:50:17Z'},
                           {'count': 132066,
                            'count_tot': 159600,
                            'mean': 49.70755256053988,
                            'time': '2017-01-28T09:58:11Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'mean': None,
                            'time': '2017-01-30T10:46:34Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon_with_all_agg_methods(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Polygon", coordinates=[[
                                            [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                        ]]),
                                        agg_methods=['mean', 'median', 'std', 'min', 'max', 'count'])

        expected_result = [{'count': 122392,
                            'count_tot': 159600,
                            'max': 166.57278442382812,
                            'mean': 56.12519223634024,
                            'median': 48.009796142578125,
                            'min': 0.02251400426030159,
                            'std': 40.78859862094861,
                            'time': '2017-01-16T10:09:22Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'max': None,
                            'mean': None,
                            'median': None,
                            'min': None,
                            'std': None,
                            'time': '2017-01-25T09:35:51Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'max': None,
                            'mean': None,
                            'median': None,
                            'min': None,
                            'std': None,
                            'time': '2017-01-26T10:50:17Z'},
                           {'count': 132066,
                            'count_tot': 159600,
                            'max': 158.7908477783203,
                            'mean': 49.70755256053988,
                            'median': 39.326446533203125,
                            'min': 0.02607600949704647,
                            'std': 34.98868194514787,
                            'time': '2017-01-28T09:58:11Z'},
                           {'count': 0,
                            'count_tot': 159600,
                            'max': None,
                            'mean': None,
                            'median': None,
                            'min': None,
                            'std': None,
                            'time': '2017-01-30T10:46:34Z'}]

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon_one_valid(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Polygon", coordinates=[[
                                            [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                        ]]),
                                        agg_methods=['mean', 'count'],
                                        max_valids=1)
        expected_result = [{'count': 132066,
                            'count_tot': 159600,
                            'mean': 49.70755256053988,
                            'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon_only_valids(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx, 'demo', 'conc_tsm',
                                        dict(type="Polygon", coordinates=[[
                                            [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                        ]]),
                                        agg_methods=['mean', 'count'],
                                        max_valids=-1)
        expected_result = [{'count': 122392,
                            'count_tot': 159600,
                            'mean': 56.12519223634024,
                            'time': '2017-01-16T10:09:22Z'},
                           {'count': 132066,
                            'count_tot': 159600,
                            'mean': 49.70755256053988,
                            'time': '2017-01-28T09:58:11Z'}]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_collection(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx,
                                        'demo', 'conc_tsm',
                                        dict(type="GeometryCollection",
                                             geometries=[
                                                 dict(type="Point", coordinates=[2.1, 51.4])]),
                                        agg_methods=['mean', 'count'],
                                        start_date=np.datetime64('2017-01-15'),
                                        end_date=np.datetime64('2017-01-29'))
        expected_result = [[{'mean': 3.534773588180542, 'time': '2017-01-16T10:09:22Z'},
                            {'mean': None, 'time': '2017-01-25T09:35:51Z'},
                            {'mean': None, 'time': '2017-01-26T10:50:17Z'},
                            {'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z'}]]
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_polygon_collection(self):
        ctx = new_test_service_context()
        actual_result = get_time_series(ctx,
                                        'demo', 'conc_tsm',
                                        dict(type="GeometryCollection",
                                             geometries=[dict(type="Polygon", coordinates=[[
                                                 [1., 51.], [2., 51.], [2., 52.], [1., 52.],
                                                 [1., 51.]
                                             ]])]),
                                        agg_methods=['mean', 'count'])
        expected_result = [[{'count': 122392,
                             'count_tot': 159600,
                             'mean': 56.12519223634024,
                             'time': '2017-01-16T10:09:22Z'},
                            {'count': 0,
                             'count_tot': 159600,
                             'mean': None,
                             'time': '2017-01-25T09:35:51Z'},
                            {'count': 0,
                             'count_tot': 159600,
                             'mean': None,
                             'time': '2017-01-26T10:50:17Z'},
                            {'count': 132066,
                             'count_tot': 159600,
                             'mean': 49.70755256053988,
                             'time': '2017-01-28T09:58:11Z'},
                            {'count': 0,
                             'count_tot': 159600,
                             'mean': None,
                             'time': '2017-01-30T10:46:34Z'}]]

        self.assertAlmostEqualDeep(expected_result, actual_result)


class CollectTimeSeriesResultTest(unittest.TestCase, AlmostEqualDeepMixin):

    def test_all_types_converted_correctly(self):
        self.maxDiff = None

        time_series_ds = xr.Dataset(
            data_vars=dict(time=xr.DataArray(pd.date_range(start='2010-04-05', periods=4, freq='1D'), dims='time'),
                           a=xr.DataArray([True, False, False, True], dims='time'),
                           b=xr.DataArray([32, 33, 35, 34], dims='time'),
                           c=xr.DataArray([0.4, 0.2, np.nan, 0.7], dims='time')))
        result = _collect_timeseries_result(time_series_ds, {'a': 'a', 'b': 'b', 'c': 'c'}, max_valids=None)
        self.assertEqual([{'a': True, 'b': 32, 'c': 0.4, 'time': '2010-04-05T00:00:00Z'},
                          {'a': False, 'b': 33, 'c': 0.2, 'time': '2010-04-06T00:00:00Z'},
                          {'a': False, 'b': 35, 'c': None, 'time': '2010-04-07T00:00:00Z'},
                          {'a': True, 'b': 34, 'c': 0.7, 'time': '2010-04-08T00:00:00Z'}],
                         result)
        self.assertEqual('[{"a": true, "b": 32, "c": 0.4, "time": "2010-04-05T00:00:00Z"},'
                         ' {"a": false, "b": 33, "c": 0.2, "time": "2010-04-06T00:00:00Z"},'
                         ' {"a": false, "b": 35, "c": null, "time": "2010-04-07T00:00:00Z"},'
                         ' {"a": true, "b": 34, "c": 0.7, "time": "2010-04-08T00:00:00Z"}]',
                         json.dumps(result))

    def test_num_valids(self):
        self.maxDiff = None

        time_series_ds = xr.Dataset(
            data_vars=dict(time=xr.DataArray(pd.date_range(start='2010-04-05', periods=5, freq='1D'), dims='time'),
                           a=xr.DataArray([np.nan, 5, np.nan, 2, 3], dims='time'),
                           b=xr.DataArray([np.nan, 33, np.nan, np.nan, 23], dims='time'),
                           c=xr.DataArray([np.nan, 0.2, np.nan, 0.7, np.nan], dims='time')))

        result = _collect_timeseries_result(time_series_ds, {'a': 'a', 'b': 'b', 'c': 'c'}, max_valids=-1)
        self.assertEqual([{'a': 5.0, 'b': 33.0, 'c': 0.2, 'time': '2010-04-06T00:00:00Z'},
                          {'a': 2.0, 'b': None, 'c': 0.7, 'time': '2010-04-08T00:00:00Z'},
                          {'a': 3.0, 'b': 23.0, 'c': None, 'time': '2010-04-09T00:00:00Z'}],
                         result)

        result = _collect_timeseries_result(time_series_ds, {'a': 'a', 'b': 'b', 'c': 'c'}, max_valids=2)
        self.assertEqual([{'a': 2.0, 'b': None, 'c': 0.7, 'time': '2010-04-08T00:00:00Z'},
                          {'a': 3.0, 'b': 23.0, 'c': None, 'time': '2010-04-09T00:00:00Z'}],
                         result)

    def test_count(self):
        self.maxDiff = None

        time_series_ds = xr.Dataset(
            data_vars=dict(time=xr.DataArray(pd.date_range(start='2010-04-05', periods=3, freq='1D'), dims='time'),
                           c=xr.DataArray([23, 78, 74], dims='time')),
            attrs=dict(max_number_of_observations=82))

        result = _collect_timeseries_result(time_series_ds, {'count': 'c'}, max_valids=-1)
        self.assertEqual([{'count': 23, 'count_tot': 82, 'time': '2010-04-05T00:00:00Z'},
                          {'count': 78, 'count_tot': 82, 'time': '2010-04-06T00:00:00Z'},
                          {'count': 74, 'count_tot': 82, 'time': '2010-04-07T00:00:00Z'}],
                         result)


@unittest.skipUnless(os.environ.get('XCUBE_TS_PERF_TEST') == '1', 'XCUBE_TS_PERF_TEST is not 1')
class TsPerfTest(unittest.TestCase):

    def test_point_ts_perf(self):
        TEST_CUBE = 'ts_test.zarr'

        if not os.path.isdir(TEST_CUBE):
            from xcube.core.new import new_cube
            cube = new_cube(time_periods=2000, variables=dict(analysed_sst=280.4))
            cube = cube.chunk(dict(time=1, lon=90, lat=90))
            cube.to_zarr(TEST_CUBE)

        ctx = ServiceContext(
            base_dir='.',
            config=dict(
                Datasets=[
                    dict(Identifier='ts_test',
                         FileSystem='file',
                         Path=TEST_CUBE,
                         Format='zarr')
                ]
            ))

        N = 5
        import random
        import time
        time_sum = 0.0
        for i in range(N):
            lon = -180 + 360 * random.random()
            lat = -90 + 180 * random.random()

            t1 = time.perf_counter()
            result = get_time_series(ctx, 'ts_test', 'analysed_sst', dict(type='Point', coordinates=[lon, lat]))
            t2 = time.perf_counter()

            self.assertIsInstance(result, list)
            self.assertEqual(2000, len(result))

            time_delta = t2 - t1
            time_sum += time_delta
            print(f'test {i + 1} took {time_delta} seconds')

        print(f'all tests took {time_sum / N} seconds in average')


# test/webapi/controllers/test_ts_legacy.py

import unittest

import numpy as np

from test.webapi.helpers import new_test_service_context
from xcube.webapi.controllers.ts_legacy import get_time_series_info, get_time_series_for_point, \
    get_time_series_for_geometry, get_time_series_for_geometry_collection
from test.mixins import AlmostEqualDeepMixin


class TsLegacyControllerTest(unittest.TestCase, AlmostEqualDeepMixin):

    def setUp(self) -> None:
        self.maxDiff = None

    def test_get_time_series_for_point_invalid_lat_and_lon(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo', 'conc_tsm',
                                                  lon=-150.0, lat=-30.0)
        expected_result = {'results': []}
        self.assertEqual(expected_result, actual_result)

    def test_get_time_series_for_point(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo', 'conc_tsm',
                                                  lon=2.1, lat=51.4,
                                                  start_date=np.datetime64('2017-01-15'),
                                                  end_date=np.datetime64('2017-01-29'))
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 3.534773588180542,
                                                   'totalCount': 1,
                                                   'validCount': 1}},
                                       {'date': '2017-01-25T09:35:51Z',
                                        'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                       {'date': '2017-01-26T10:50:17Z',
                                        'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_one_valid(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo', 'conc_tsm',
                                                  lon=2.1, lat=51.4,
                                                  start_date=np.datetime64('2017-01-15'),
                                                  end_date=np.datetime64('2017-01-29'),
                                                  max_valids=1)
        expected_result = {'results': [{'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_only_valids(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo', 'conc_tsm',
                                                  lon=2.1, lat=51.4,
                                                  start_date=np.datetime64('2017-01-15'),
                                                  end_date=np.datetime64('2017-01-29'),
                                                  max_valids=-1)
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 3.534773588180542,
                                                   'totalCount': 1,
                                                   'validCount': 1}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_point_with_uncertainty(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_point(ctx, 'demo-1w', 'conc_tsm',
                                                  lon=2.1, lat=51.4,
                                                  start_date=np.datetime64('2017-01-15'),
                                                  end_date=np.datetime64('2017-01-29'))
        expected_result = {'results': [{'date': '2017-01-22T00:00:00Z',
                                        'result': {'average': 3.534773588180542,
                                                   'totalCount': 1,
                                                   'validCount': 1}},
                                       {'date': '2017-01-29T00:00:00Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_point(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Point", coordinates=[2.1, 51.4]),
                                                     start_date=np.datetime64('2017-01-15'),
                                                     end_date=np.datetime64('2017-01-29'),
                                                     include_count=True)
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 3.534773588180542,
                                                   'totalCount': 1,
                                                   'validCount': 1}},
                                       {'date': '2017-01-25T09:35:51Z',
                                        'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                       {'date': '2017-01-26T10:50:17Z',
                                        'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 20.12085723876953,
                                                   'totalCount': 1,
                                                   'validCount': 1}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_polygon(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Polygon", coordinates=[[
                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                                     ]]),
                                                     include_count=True)
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 56.12519223634024,
                                                   'totalCount': 159600,
                                                   'validCount': 122392}},
                                       {'date': '2017-01-25T09:35:51Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'validCount': 0}},
                                       {'date': '2017-01-26T10:50:17Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'validCount': 0}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 49.70755256053988,
                                                   'totalCount': 159600,
                                                   'validCount': 132066}},
                                       {'date': '2017-01-30T10:46:34Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'validCount': 0}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_polygon_with_stdev(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Polygon", coordinates=[[
                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                                     ]]),
                                                     include_count=True,
                                                     include_stdev=True)

        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 56.12519223634024,
                                                   'totalCount': 159600,
                                                   'uncertainty': 40.78859862094861,
                                                   'validCount': 122392}},
                                       {'date': '2017-01-25T09:35:51Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'uncertainty': None,
                                                   'validCount': 0}},
                                       {'date': '2017-01-26T10:50:17Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'uncertainty': None,
                                                   'validCount': 0}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 49.70755256053988,
                                                   'totalCount': 159600,
                                                   'uncertainty': 34.98868194514786,
                                                   'validCount': 132066}},
                                       {'date': '2017-01-30T10:46:34Z',
                                        'result': {'average': None,
                                                   'totalCount': 159600,
                                                   'uncertainty': None,
                                                   'validCount': 0}}]}

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_polygon_one_valid(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Polygon", coordinates=[[
                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                                     ]]),
                                                     include_count=True,
                                                     max_valids=1)
        expected_result = {'results': [{'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 49.70755256053988,
                                                   'totalCount': 159600,
                                                   'validCount': 132066}}]}

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometry_polygon_only_valids(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry(ctx, 'demo', 'conc_tsm',
                                                     dict(type="Polygon", coordinates=[[
                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]
                                                     ]]),
                                                     include_count=True,
                                                     max_valids=-1)
        expected_result = {'results': [{'date': '2017-01-16T10:09:22Z',
                                        'result': {'average': 56.12519223634024,
                                                   'totalCount': 159600,
                                                   'validCount': 122392}},
                                       {'date': '2017-01-28T09:58:11Z',
                                        'result': {'average': 49.70755256053988,
                                                   'totalCount': 159600,
                                                   'validCount': 132066}}]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometries_incl_point(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry_collection(ctx,
                                                                'demo', 'conc_tsm',
                                                                dict(type="GeometryCollection",
                                                                     geometries=[
                                                                         dict(type="Point", coordinates=[2.1, 51.4])]),
                                                                start_date=np.datetime64('2017-01-15'),
                                                                end_date=np.datetime64('2017-01-29'),
                                                                include_count=True)
        expected_result = {'results': [[{'date': '2017-01-16T10:09:22Z',
                                         'result': {'average': 3.534773588180542,
                                                    'totalCount': 1,
                                                    'validCount': 1}},
                                        {'date': '2017-01-25T09:35:51Z',
                                         'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                        {'date': '2017-01-26T10:50:17Z',
                                         'result': {'average': None, 'totalCount': 1, 'validCount': 0}},
                                        {'date': '2017-01-28T09:58:11Z',
                                         'result': {'average': 20.12085723876953,
                                                    'totalCount': 1,
                                                    'validCount': 1}}]]}
        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_for_geometries_incl_polygon(self):
        ctx = new_test_service_context()
        actual_result = get_time_series_for_geometry_collection(ctx,
                                                                'demo', 'conc_tsm',
                                                                dict(type="GeometryCollection",
                                                                     geometries=[dict(type="Polygon", coordinates=[[
                                                                         [1., 51.], [2., 51.], [2., 52.], [1., 52.],
                                                                         [1., 51.]
                                                                     ]])]),
                                                                include_count=True)
        expected_result = {'results': [[{'date': '2017-01-16T10:09:22Z',
                                         'result': {'average': 56.12519223634024,
                                                    'totalCount': 159600,
                                                    'validCount': 122392}},
                                        {'date': '2017-01-25T09:35:51Z',
                                         'result': {'average': None,
                                                    'totalCount': 159600,
                                                    'validCount': 0}},
                                        {'date': '2017-01-26T10:50:17Z',
                                         'result': {'average': None,
                                                    'totalCount': 159600,
                                                    'validCount': 0}},
                                        {'date': '2017-01-28T09:58:11Z',
                                         'result': {'average': 49.70755256053988,
                                                    'totalCount': 159600,
                                                    'validCount': 132066}},
                                        {'date': '2017-01-30T10:46:34Z',
                                         'result': {'average': None,
                                                    'totalCount': 159600,
                                                    'validCount': 0}}]]}

        self.assertAlmostEqualDeep(expected_result, actual_result)

    def test_get_time_series_info(self):
        self.maxDiff = None

        ctx = new_test_service_context()
        info = get_time_series_info(ctx)

        expected_result = self._get_expected_info_dict()
        self.assertEqual(expected_result, info)

    @staticmethod
    def _get_expected_info_dict():
        expected_dict = {'layers': []}
        bounds = {'xmin': 0.0, 'ymin': 50.0,
                  'xmax': 5.0, 'ymax': 52.5}
        demo_times = ['2017-01-16T10:09:22Z',
                      '2017-01-25T09:35:51Z',
                      '2017-01-26T10:50:17Z',
                      '2017-01-28T09:58:11Z',
                      '2017-01-30T10:46:34Z']
        demo_variables = ['c2rcc_flags',
                          'conc_chl',
                          'conc_tsm',
                          'kd489',
                          'quality_flags']
        for demo_variable in demo_variables:
            dict_variable = {'name': f'demo.{demo_variable}', 'dates': demo_times, 'bounds': bounds}
            expected_dict['layers'].append(dict_variable)
        demo1w_times = ['2017-01-22T00:00:00Z', '2017-01-29T00:00:00Z', '2017-02-05T00:00:00Z']
        for demo_variable in demo_variables:
            dict_variable = {'name': f'demo-1w.{demo_variable}', 'dates': demo1w_times, 'bounds': bounds}
            expected_dict['layers'].append(dict_variable)
            dict_variable = {'name': f'demo-1w.{demo_variable}_stdev', 'dates': demo1w_times, 'bounds': bounds}
            expected_dict['layers'].append(dict_variable)
        return expected_dict


# test/webapi/datasets/__init__.py

#  The MIT License (MIT)
#  Copyright (c) 2022 by the xcube development team and contributors
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.


# test/webapi/datasets/test_authutil.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest

from xcube.server.api import ApiError
from xcube.webapi.datasets.authutil import check_scopes, assert_scopes


class DatasetScopesTest(unittest.TestCase):

    def test_check_scopes_ok(self):
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         None,
                         is_substitute=True)
        )
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test1.zarr'})
        )
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test?.zarr'})
        )
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test1.*'})
        )

    def test_check_scopes_fails(self):
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test1.zarr'},
                         None,
                         is_substitute=False)
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test1.zarr'},
                         set(),
                         is_substitute=True)
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test1.zarr'},
                         is_substitute=True)
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test2.zarr'},
                         {'read:dataset:test1.zarr'})
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test2.zarr'},
                         {'read:dataset:test1.zarr'},
                         is_substitute=True)
        )

    def test_assert_scopes_ok(self):
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test1.zarr'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:*'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test?.zarr'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test1.*'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test2.zarr',
                       'read:dataset:test3.zarr',
                       'read:dataset:test1.zarr'})

    def test_assert_scopes_fails(self):
        with self.assertRaises(ApiError.Unauthorized) as cm:
            assert_scopes({'read:dataset:test1.zarr'},
                          None)
        self.assertEquals(
            'HTTP status 401: Missing permission "read:dataset:test1.zarr"',
            f'{cm.exception}'
        )

        with self.assertRaises(ApiError.Unauthorized) as cm:
            assert_scopes({'read:dataset:test1.zarr'},
                          set())
        self.assertEquals(
            'HTTP status 401: Missing permission "read:dataset:test1.zarr"',
            f'{cm.exception}'
        )

        with self.assertRaises(ApiError.Unauthorized) as cm:
            assert_scopes({'read:dataset:test1.zarr'},
                          {'read:dataset:test2.zarr'})
        self.assertEquals(
            'HTTP status 401: Missing permission "read:dataset:test1.zarr"',
            f'{cm.exception}'
        )


# test/webapi/datasets/test_context.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest
from typing import Union

from test.webapi.helpers import get_api_ctx
from xcube.server.api import Context, ServerConfig
from xcube.webapi.datasets.context import DatasetsContext


def get_datasets_ctx(
        server_config: Union[str, ServerConfig] = "config.yml"
) -> DatasetsContext:
    return get_api_ctx("datasets", DatasetsContext, server_config)


class DatasetsContextTest(unittest.TestCase):

    def test_ctx_ok(self):
        ctx = get_datasets_ctx()
        self.assertIsInstance(ctx.server_ctx, Context)
        self.assertIsInstance(ctx.auth_ctx, Context)
        self.assertIsInstance(ctx.places_ctx, Context)


# test/webapi/datasets/test_controllers.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest

from xcube.webapi.datasets.controllers import find_dataset_places
from .test_context import get_datasets_ctx


class DatasetsControllersTest(unittest.TestCase):

    def test_find_dataset_features(self):
        ctx = get_datasets_ctx()
        places = find_dataset_places(ctx, "all", "demo",
                                     "http://localhost:8080")
        self._assertPlaceGroup(places, 3, {'0', '1', '2'})

    def _assertPlaceGroup(self, feature_collection, expected_count,
                          expected_ids):
        self.assertIsInstance(feature_collection, dict)
        self.assertIn("type", feature_collection)
        self.assertEqual("FeatureCollection", feature_collection["type"])
        self.assertIn("features", feature_collection)
        features = feature_collection["features"]
        self.assertIsInstance(features, list)
        self.assertEqual(expected_count, len(features))
        actual_ids = {f["id"] for f in features if "id" in f}
        self.assertEqual(expected_ids, actual_ids)


# test/webapi/places/__init__.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.


# test/webapi/places/test_context.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest

from test.webapi.helpers import get_api_ctx
from xcube.server.api import Context
from xcube.webapi.places.context import PlacesContext


def get_places_ctx() -> PlacesContext:
    return get_api_ctx("places", PlacesContext, "config.yml")


class PlacesContextTest(unittest.TestCase):

    def test_ctx_ok(self):
        ctx = get_places_ctx()
        self.assertIsInstance(ctx.server_ctx, Context)
        self.assertIsInstance(ctx.auth_ctx, Context)


# test/webapi/places/test_controllers.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import unittest

from xcube.server.api import ApiError
from xcube.webapi.places.controllers import find_places
from .test_context import get_places_ctx

BAD_REQUEST_MSG = ("HTTP status 400:"
                   " Received invalid geometry bbox,"
                   " geometry WKT, or GeoJSON object")


class PlacesControllersTest(unittest.TestCase):

    def test_find_places_all(self):
        ctx = get_places_ctx()
        places = find_places(ctx, "all", "http://localhost:9090")
        self._assertPlaceGroup(places, 6, {'0', '1', '2', '3', '4', '5'})

    def test_find_places_by_box_ok(self):
        ctx = get_places_ctx()
        places = find_places(ctx, "all", "http://localhost:9090",
                             query_geometry="-1,49,2,55")
        self._assertPlaceGroup(places, 2, {'0', '3'})

    def test_find_places_by_box_fail(self):
        ctx = get_places_ctx()

        with self.assertRaises(ApiError.BadRequest) as cm:
            find_places(
                ctx, "all", "http://localhost:9090",
                query_geometry="-1,49,55"
            )
        self.assertEqual(BAD_REQUEST_MSG, f"{cm.exception}")

        with self.assertRaises(ApiError.BadRequest) as cm:
            find_places(
                ctx, "all", "http://localhost:9090",
                query_geometry="-1,49,x,55"
            )
        self.assertEqual(BAD_REQUEST_MSG, f"{cm.exception}")

    def test_find_places_by_wkt_ok(self):
        ctx = get_places_ctx()
        places = find_places(
            ctx, "all", "http://localhost:9090",
            query_geometry="POLYGON ((2 49, 2 55, -1 55, -1 49, 2 49))"
        )
        self._assertPlaceGroup(places, 2, {'0', '3'})

    def test_find_places_by_wkt_fail(self):
        ctx = get_places_ctx()

        with self.assertRaises(ApiError.BadRequest) as cm:
            find_places(
                ctx, "all", "http://localhost:9090",
                query_geometry="POLYGLON ((2 49, 2 55, -1 55, -1 49, 2 49))"
            )
        self.assertEqual(BAD_REQUEST_MSG, f"{cm.exception}")

    def test_find_places_by_geojson_ok(self):
        ctx = get_places_ctx()

        geojson_obj = {'type': 'Polygon',
                       'coordinates': ((
                                           (2.0, 49.0), (2.0, 55.0),
                                           (-1.0, 55.0),
                                           (-1.0, 49.0), (2.0, 49.0)),)}
        places = find_places(ctx, "all", "http://localhost:9090",
                             query_geometry=geojson_obj)
        self._assertPlaceGroup(places, 2, {'0', '3'})

        geojson_obj = {'type': 'Feature', 'properties': {},
                       'geometry': geojson_obj}
        places = find_places(ctx, "all", "http://localhost:9090",
                             query_geometry=geojson_obj)
        self._assertPlaceGroup(places, 2, {'0', '3'})

        geojson_obj = {'type': 'FeatureCollection', 'features': [geojson_obj]}
        places = find_places(ctx, "all", "http://localhost:9090",
                             query_geometry=geojson_obj)
        self._assertPlaceGroup(places, 2, {'0', '3'})

    def test_find_places_by_geojson_fail(self):
        ctx = get_places_ctx()
        with self.assertRaises(ApiError.BadRequest) as cm:
            geojson_obj = {'type': 'FeatureCollection', 'features': []}
            find_places(ctx, "all", "http://localhost:9090",
                        query_geometry=geojson_obj)
        self.assertEqual(BAD_REQUEST_MSG, f"{cm.exception}")

    def _assertPlaceGroup(self, feature_collection, expected_count,
                          expected_ids):
        self.assertIsInstance(feature_collection, dict)
        self.assertIn("type", feature_collection)
        self.assertEqual("FeatureCollection", feature_collection["type"])
        self.assertIn("features", feature_collection)
        features = feature_collection["features"]
        self.assertIsInstance(features, list)
        self.assertEqual(expected_count, len(features))
        actual_ids = {f["id"] for f in features if "id" in f}
        self.assertEqual(expected_ids, actual_ids)


# test/webapi/places/test_routes.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import json

from ..helpers import ServerTest


class PlacesRoutesTest(ServerTest):

    def test_places(self):
        result, status = self.fetch_json('/places')
        self.assertEqual(200, status)
        for pg in result['placeGroups']:
            pg['sourcePaths'] = []
        self.assertEqual(
            {'placeGroups': [
                {'features': None,
                 'id': 'inside-cube',
                 'join': None,
                 'propertyMapping': None,
                 'sourceEncoding': 'utf-8',
                 'sourcePaths': [],
                 'title': 'Points inside the cube',
                 'type': 'FeatureCollection'},
                {'features': None,
                 'id': 'outside-cube',
                 'join': None,
                 'propertyMapping': None,
                 'sourceEncoding': 'utf-8',
                 'sourcePaths': [],
                 'title': 'Points outside the cube',
                 'type': 'FeatureCollection'}
            ]},
            result
        )

    def test_place_by_id(self):
        expected = {
            'places': {
                'features': [
                    {'geometry': {'coordinates': [1.5, 52.1],
                                  'type': 'Point'},
                     'id': '0',
                     'properties': {'ID': '1',
                                    'Name': 'Station 1',
                                    'Region_Name': 'Belgium',
                                    'Sub_Region_Name': 'Inside'},
                     'type': 'Feature'},
                    {'geometry': {'coordinates': [2.5, 51.5],
                                  'type': 'Point'},
                     'id': '1',
                     'properties': {'ID': '2',
                                    'Name': 'Station 2',
                                    'Region_Name': 'Belgium',
                                    'Sub_Region_Name': 'Inside'},
                     'type': 'Feature'},
                    {'geometry': {'coordinates': [4.5, 50.9],
                                  'type': 'Point'},
                     'id': '2',
                     'properties': {'ID': '3',
                                    'Name': 'Station 3',
                                    'Region_Name': 'Belgium',
                                    'Sub_Region_Name': 'Inside'},
                     'type': 'Feature'}
                ],
                'type': 'FeatureCollection'
            }
        }

        bbox = "0,40,20,60"
        result, status = self.fetch_json(f'/places/inside-cube?bbox={bbox}')
        self.assertEqual(200, status)
        self.assertEqual(expected, result)

        geom = {
            "type": "Polygon",
            "coordinates": [[
                [0, 40],
                [0, 60],
                [20, 60],
                [20, 40],
                [0, 40]
            ]]
        }
        bbox = bytes(json.dumps(geom), 'utf-8')

        result, status = self.fetch_json('/places/inside-cube',
                                         method="POST",
                                         body=bbox)
        self.assertEqual(200, status)
        self.assertEqual(expected, result)


# test/webapi/s3/__init__.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.


# test/webapi/s3/test_routes.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import urllib3.response

from ..helpers import ServerTest


class S3RoutesTest(ServerTest):
    def assertResponseOK(self, response: urllib3.response.HTTPResponse):
        self.assertEqual(200, response.status)

    def assertResourceNotFoundResponse(
            self, response: urllib3.response.HTTPResponse
    ):
        self.assertEqual(404, response.status)

    def test_fetch_list_s3bucket(self):
        response = self.fetch('/s3bucket')
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket?delimiter=%2F')
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket?delimiter=%2F&prefix=demo%2F')
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket?delimiter=%2F&list-type=2')
        self.assertResponseOK(response)
        response = self.fetch(
            '/s3bucket?delimiter=%2F&prefix=demo%2F&list-type=2')
        self.assertResponseOK(response)

    def test_fetch_head_s3bucket_object(self):
        self._assert_fetch_head_s3bucket_object(method='HEAD')

    def test_fetch_get_s3bucket_object(self):
        self._assert_fetch_head_s3bucket_object(method='GET')

    def _assert_fetch_head_s3bucket_object(self, method):
        response = self.fetch('/s3bucket/demo', method=method)
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket/demo/', method=method)
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket/demo/.zattrs', method=method)
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket/demo/.zgroup', method=method)
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket/demo/.zarray', method=method)
        self.assertResourceNotFoundResponse(response)
        response = self.fetch('/s3bucket/demo/time/.zattrs', method=method)
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket/demo/time/.zarray', method=method)
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket/demo/time/.zgroup', method=method)
        self.assertResourceNotFoundResponse(response)
        response = self.fetch('/s3bucket/demo/time/0', method=method)
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket/demo/conc_chl/.zattrs',
                              method=method)
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket/demo/conc_chl/.zarray',
                              method=method)
        self.assertResponseOK(response)
        response = self.fetch('/s3bucket/demo/conc_chl/.zgroup',
                              method=method)
        self.assertResourceNotFoundResponse(response)
        response = self.fetch('/s3bucket/demo/conc_chl/3.2.4', method=method)
        self.assertResponseOK(response)


# test/__init__.py

from test.util.test_plugin import init_plugin

# xcube.util.plugin:discover_plugin_modules() will search for "init_test" first (because this package is called "test").
# Only if "init_test" is not found it would search for "init_plugin".
init_test = init_plugin


# test/mixins.py

from typing import Any
import array

# noinspection PyUnresolvedReferences
class AlmostEqualDeepMixin:

    def assertAlmostEqualDeep(self,
                              first: Any,
                              second: Any,
                              msg: str = None,
                              places: int = None,
                              delta: float = None):
        """
        Performs self.assertAlmostEqual() on number, and numbers in dictionaries and sequences.

        Note that decimal places (from zero) are usually not the same
        as significant digits (measured from the most significant digit).

        If the two objects compare equal then they will automatically
        compare almost equal.
        """
        if first == second:
            # shortcut
            return

        if isinstance(first, float) or isinstance(second, float):
            self.assertAlmostEqual(first, second, msg=msg, places=places, delta=delta)
            # success
            return

        if isinstance(first, set) or isinstance(second, set):
            # we currently cannot almost-equal-compare sets, but it is possible:
            #  - first remove items from both sets that are strictly equal
            #  - secondly remove items from both sets that are almost equal
            #  - are both empty in the end?
            self.assertEqual(first, second, msg=msg)
            # success
            return

        if isinstance(first, dict) and isinstance(second, dict):
            if len(first) != len(second):
                self.assertEqual(first, second, msg=msg)  # fail
            for key, first_value in first.items():
                if key not in second:
                    self.assertEqual(first, second, msg=msg)  # fail
                second_value = second[key]
                self.assertAlmostEqualDeep(first_value, second_value, msg=msg, delta=delta, places=places)
            for key, second_value in second.items():
                if key not in first:
                    self.assertEqual(first, second, msg=msg)  # fail
            # success
            return

        if isinstance(first, (list, tuple, array.array)) and isinstance(second, (list, tuple, array.array)):
            if len(first) != len(second):
                # let fail
                self.assertEqual(first, second, msg=msg)
            for first_value, second_value in zip(first, second):
                self.assertAlmostEqualDeep(first_value, second_value, msg=msg, delta=delta, places=places)
            # success
            return

        self.assertEqual(first, second, msg=msg)


# test/plugin.py

# init_plugin() is used by test/util/test_plugin

# noinspection PyUnresolvedReferences
from test.util.test_plugin import init_plugin

# test/s3test.py

import os
import subprocess
import sys
import time
import unittest
import urllib
import urllib.error
import urllib.request

import moto.server

MOTO_SERVER_ENDPOINT_URL = f'http://127.0.0.1:5000'

MOTOSERVER_PATH = moto.server.__file__
MOTOSERVER_ARGS = [sys.executable, MOTOSERVER_PATH, 's3']


class S3Test(unittest.TestCase):
    _moto_server = None

    @classmethod
    def setUpClass(cls) -> None:

        """Mocked AWS Credentials for moto."""
        os.environ['AWS_ACCESS_KEY_ID'] = 'testing'
        os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing'
        os.environ['AWS_SECURITY_TOKEN'] = 'testing'
        os.environ['AWS_SESSION_TOKEN'] = 'testing'

        cls._moto_server = subprocess.Popen(MOTOSERVER_ARGS)
        t0 = time.perf_counter()
        running = False
        while not running and time.perf_counter() - t0 < 60.0:
            try:
                with urllib.request.urlopen(MOTO_SERVER_ENDPOINT_URL, timeout=1.0):
                    running = True
                    print(f'moto_server started after {round(1000 * (time.perf_counter() - t0))} ms')

            except urllib.error.URLError as e:
                pass
        if not running:
            raise Exception(f'Failed to start moto server after {round(1000 * (time.perf_counter() - t0))} ms')

    def setUp(self) -> None:
        # see https://github.com/spulec/moto/issues/2288
        urllib.request.urlopen(urllib.request.Request(MOTO_SERVER_ENDPOINT_URL + '/moto-api/reset', method='POST'))

    @classmethod
    def tearDownClass(cls) -> None:
        cls._moto_server.kill()


# test/sampledata.py

import numpy as np
import pandas as pd
import xarray as xr


def new_test_dataset(time, height=180, **indexers):
    """
    Create a test dataset with dimensions ("time", "lat", "lon") and data variables given by *indexers*.

    :param time: Single date/time string or sequence of date-time strings.
    :param height: Size of the latitude dimension.
    :param indexers: Variable name to value mapping.
           Value may be a scalar or a vector of same length as *time*.
    :return: test dataset
    """
    # TODO (forman): get rid of this code here, utilise xcube.core.new_cube() instead
    time = [time] if isinstance(time, str) else time
    width = height * 2
    num_times = len(time)
    res = 180 / height
    shape = (1, height, width)
    data_vars = dict()
    for name, value in indexers.items():
        try:
            values = list(value)
        except TypeError:
            values = [value] * num_times
        if len(values) != num_times:
            raise ValueError()
        data_vars[name] = (['time', 'lat', 'lon'],
                           np.concatenate(tuple(np.full(shape, values[i]) for i in range(num_times))))
    time = np.array(pd.to_datetime(time), dtype=np.datetime64)
    return xr.Dataset(data_vars,
                      coords=dict(time=(
                          ['time'], time, dict(units='nanoseconds since 1970-01-01', calendar='proleptic_gregorian')),
                          lat=(['lat'], np.linspace(-90 + res, +90 - res, height)),
                          lon=(['lon'], np.linspace(-180 + res, +180 - res, width))))


def create_s2plus_dataset():
    x = xr.DataArray([310005., 310015., 310025., 310035., 310045.], dims=["x"],
                     attrs=dict(units="m", standard_name="projection_x_coordinate"))
    y = xr.DataArray([5689995., 5689985., 5689975., 5689965., 5689955.], dims=["y"],
                     attrs=dict(units="m", standard_name="projection_y_coordinate"))
    lon = xr.DataArray([[0.272763, 0.272906, 0.273050, 0.273193, 0.273336],
                        [0.272768, 0.272911, 0.273055, 0.273198, 0.273342],
                        [0.272773, 0.272917, 0.273060, 0.273204, 0.273347],
                        [0.272779, 0.272922, 0.273066, 0.273209, 0.273352],
                        [0.272784, 0.272927, 0.273071, 0.273214, 0.273358]],
                       dims=["y", "x"], attrs=dict(units="degrees_east", standard_name="longitude"))
    lat = xr.DataArray([[51.329464, 51.329464, 51.329468, 51.32947, 51.329475],
                        [51.329372, 51.329376, 51.32938, 51.329384, 51.329388],
                        [51.329285, 51.329285, 51.32929, 51.329292, 51.329296],
                        [51.329193, 51.329197, 51.32920, 51.329205, 51.329205],
                        [51.329100, 51.329105, 51.32911, 51.329113, 51.329117]],
                       dims=["y", "x"], attrs=dict(units="degrees_north", standard_name="latitude"))
    rrs_443 = xr.DataArray([[0.014000, 0.014000, 0.016998, 0.016998, 0.016998],
                            [0.014000, 0.014000, 0.016998, 0.016998, 0.016998],
                            [0.019001, 0.019001, 0.016998, 0.016998, 0.016998],
                            [0.019001, 0.019001, 0.016998, 0.016998, 0.016998],
                            [0.019001, 0.019001, 0.016998, 0.016998, 0.016998]],
                           dims=["y", "x"], attrs=dict(units="sr-1", grid_mapping="transverse_mercator"))
    rrs_665 = xr.DataArray([[0.025002, 0.019001, 0.008999, 0.012001, 0.022999],
                            [0.028000, 0.021000, 0.009998, 0.008999, 0.022999],
                            [0.036999, 0.022999, 0.007999, 0.008999, 0.023998],
                            [0.041000, 0.022999, 0.007000, 0.009998, 0.021000],
                            [0.033001, 0.018002, 0.007999, 0.008999, 0.021000]],
                           dims=["y", "x"], attrs=dict(units="sr-1", grid_mapping="transverse_mercator"))
    transverse_mercator = xr.DataArray(np.array([0xffffffff], dtype=np.uint32),
                                       attrs=dict(grid_mapping_name="transverse_mercator",
                                                  scale_factor_at_central_meridian=0.9996,
                                                  longitude_of_central_meridian=3.0,
                                                  latitude_of_projection_origin=0.0,
                                                  false_easting=500000.0,
                                                  false_northing=0.0,
                                                  semi_major_axis=6378137.0,
                                                  inverse_flattening=298.257223563))
    return xr.Dataset(dict(rrs_443=rrs_443, rrs_665=rrs_665, transverse_mercator=transverse_mercator),
                      coords=dict(x=x, y=y, lon=lon, lat=lat),
                      attrs={
                          "title": "T31UCS_20180802T105621",
                          "conventions": "CF-1.6",
                          "institution": "VITO",
                          "product_type": "DCS4COP Sentinel2 Product",
                          "origin": "Copernicus Sentinel Data",
                          "project": "DCS4COP",
                          "time_coverage_start": "2018-08-02T10:59:38.888000Z",
                          "time_coverage_end": "2018-08-02T10:59:38.888000Z"
                      })


def create_highroc_dataset(no_spectra=False):
    """
    Simulates a HIGHROC OLCI L2 product in NetCDF 4 format
    """
    lon = np.array([[8, 9.3, 10.6, 11.9],
                    [8, 9.2, 10.4, 11.6],
                    [8, 9.1, 10.2, 11.3]], dtype=np.float32)
    lat = np.array([[56, 56.1, 56.2, 56.3],
                    [55, 55.2, 55.4, 55.6],
                    [54, 54.3, 54.6, 54.9]], dtype=np.float32)

    if not no_spectra:
        wavelengths = [(1, 400.0), (2, 412.5), (3, 442.5), (4, 490.0), (5, 510.0),
                       (6, 560.0), (7, 620.0), (8, 665.0), (9, 673.75), (10, 681.25),
                       (11, 708.75), (12, 753.75), (16, 778.75), (17, 865.0), (18, 885.0), (21, 940.0)]
        rtoa_desc = "Top-of-atmosphere reflectance"
        rrs_desc = "Atmospherically corrected angular dependent remote sensing reflectances"
        rtoa_vars = {f'rtoa_{i}': create_waveband(i, wl, '1', rtoa_desc) for i, wl in wavelengths}
        rrs_vars = {f'rrs_{i}': create_waveband(i, wl, 'sr^-1', rrs_desc) for i, wl in wavelengths}
    else:
        rtoa_vars = {}
        rrs_vars = {}

    return xr.Dataset(
        data_vars=dict(
            conc_chl=create_conc_chl(),
            c2rcc_flags=create_c2rcc_flag_var(),
            lon=(('y', 'x'), lon, dict(
                long_name="longitude",
                units="degrees_east",
            )),
            lat=(('y', 'x'), lat, dict(
                long_name="latitude",
                units="degrees_north",
            )),
            **rtoa_vars,
            **rrs_vars,
        ),
        attrs=dict(start_date='14-APR-2017 10:27:50.183264',
                   stop_date='14-APR-2017 10:31:42.736226')
    )


def create_waveband(index, wavelength, units, long_name=None):
    data = np.array([[7, 11, np.nan, 5],
                     [5, 10, 2, 21],
                     [16, 6, 20, 17]], dtype=np.float32)
    return (('y', 'x'), data, dict(
        long_name=long_name,
        units=units,
        spectral_band_index=index,
        wavelength=wavelength,
        bandwidth=15.0,
        valid_pixel_expression="c2rcc_flags.F1",
        _FillValue=np.nan,
    ))


def create_conc_chl():
    data = np.array([[7, 11, np.nan, 5],
                     [5, 10, 2, 21],
                     [16, 6, 20, 17]], dtype=np.float32)
    return (('y', 'x'), data, dict(
        long_name="Chlorophyll concentration",
        units="mg m^-3",
        _FillValue=np.nan,
        valid_pixel_expression="c2rcc_flags.F1",
    ))


def create_c2rcc_flag_var():
    data = np.array([[1, 1, 1, 1],
                     [1, 4, 1, 2],
                     [8, 1, 1, 1]], dtype=np.uint32)
    return xr.DataArray(data, dims=('y', 'x'), name='c2rcc_flags', attrs=dict(
        long_name="C2RCC quality flags",
        _Unsigned="true",
        flag_meanings="F1 F2 F3 F4",
        flag_masks=np.array([1, 2, 4, 8], np.int32),
        flag_coding_name="c2rcc_flags",
        flag_descriptions="D1 D2 D3 D4",
    ))


def create_cmems_sst_flag_var():
    sea = 1
    land = 2
    lake = 4
    ice = 8
    data = np.array([[[sea + ice, land + ice, lake + ice, lake],
                      [sea + ice, sea, land, land],
                      [sea, sea, sea, land]]], dtype=np.float32)
    return xr.DataArray(data, dims=('time', 'lat', 'lon'), name='mask', attrs=dict(
        long_name="land sea ice lake bit mask",
        flag_masks="0b, 1b, 2b, 3b",
        flag_meanings="sea land lake ice",
        valid_min=0,
        valid_max=12,
    ))


class SourceDatasetMixin:

    @classmethod
    def new_2x2_dataset_with_irregular_coords(cls):
        lon = np.array([[1.0, 6.0],
                        [0.0, 2.0]])
        lat = np.array([[56.0, 53.0],
                        [52.0, 50.0]])
        rad = np.array([[1.0, 2.0],
                        [3.0, 4.0]])
        return xr.Dataset(dict(lon=xr.DataArray(lon, dims=('y', 'x')),
                               lat=xr.DataArray(lat, dims=('y', 'x')),
                               rad=xr.DataArray(rad, dims=('y', 'x'))))

    @classmethod
    def new_2x2_dataset_with_irregular_coords_antimeridian(cls):
        lon = np.array([[+179.0, -176.0],
                        [+178.0, +180.0]])
        lat = np.array([[56.0, 53.0],
                        [52.0, 50.0]])
        rad = np.array([[1.0, 2.0],
                        [3.0, 4.0]])
        return xr.Dataset(dict(lon=xr.DataArray(lon, dims=('y', 'x')),
                               lat=xr.DataArray(lat, dims=('y', 'x')),
                               rad=xr.DataArray(rad, dims=('y', 'x'))))


# test/test_mixins.py

import unittest

from test.mixins import AlmostEqualDeepMixin


class AlmostEqualDeepMixinTest(unittest.TestCase, AlmostEqualDeepMixin):
    def test_int_and_float_7_places_default(self):
        self.assertAlmostEqualDeep(0, 0.8e-8)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(0, 0.8e-7)

    def test_int(self):
        self.assertAlmostEqualDeep(45, 45)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(45, 54)

    def test_str(self):
        self.assertAlmostEqualDeep("abc", "abc")
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep("abc", "Abc")

    def test_bool(self):
        self.assertAlmostEqualDeep(True, True)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(True, False)

    def test_set(self):
        expected = {'a', 1.1256, True}
        self.assertAlmostEqualDeep(expected, expected)
        self.assertAlmostEqualDeep(expected, {'a', 1.1256, True})
        with self.assertRaises(AssertionError):
            # We currently don't test sets
            self.assertAlmostEqualDeep(expected, {'a', 1.1251, True}, places=2)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, {'a', 1.1256, False})

    def test_dict(self):
        expected = {'a': 1.1256, 'b': 5}
        self.assertAlmostEqualDeep(expected, expected)
        self.assertAlmostEqualDeep(expected, {'a': 1.1256, 'b': 5})
        self.assertAlmostEqualDeep(expected, {'a': 1.1251, 'b': 5}, places=3)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, {'a': 1.1251, 'b': 5}, places=4)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, {'a': 1.1256, 'b': 6})
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, [1, 2, 3])
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, 3456)

    def test_list(self):
        expected = ['a', 1.1256, True]
        self.assertAlmostEqualDeep(expected, expected)
        self.assertAlmostEqualDeep(expected, ['a', 1.1256, True])
        self.assertAlmostEqualDeep(expected, ('a', 1.1256, True))
        self.assertAlmostEqualDeep(expected, ['a', 1.1251, True], places=3)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, ['a', 1.1251, True], places=4)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, ['a', 1.1256, False], places=4)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, [1, 2, 3])
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, 3456)

    def test_list_dict_tuple(self):
        expected = [
            {'a': True, 'b': (1.1256, 45, True)},
            {'a': False, 'b': (2.1256, 46, False)}
        ]
        self.assertAlmostEqualDeep(expected, expected)
        self.assertAlmostEqualDeep(expected, [
            {'a': True, 'b': (1.1256, 45, True)},
            {'a': False, 'b': (2.1256, 46, False)}
        ])
        self.assertAlmostEqualDeep(expected, [
            {'a': True, 'b': (1.1251, 45, True)},
            {'a': False, 'b': (2.1259, 46, False)}
        ], places=3)
        with self.assertRaises(AssertionError):
            self.assertAlmostEqualDeep(expected, [
                {'a': True, 'b': (1.1251, 45, True)},
                {'a': False, 'b': (2.1259, 46, False)}
            ], places=4)


# test/test_plugin.py

import unittest

from xcube.plugin import init_plugin
from xcube.util.extension import ExtensionRegistry


class PluginTest(unittest.TestCase):
    def test_standard_extensions_exist(self):
        ext_reg = ExtensionRegistry()

        init_plugin(ext_reg)

        self.assertTrue(ext_reg.has_extension('xcube.core.gen.iproc', 'default'))

        self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'zarr'))
        self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'netcdf4'))
        self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'csv'))
        self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'mem'))

        self.assertTrue(ext_reg.has_extension('xcube.core.store', 'file'))
        self.assertTrue(ext_reg.has_extension('xcube.core.store', 'memory'))
        self.assertTrue(ext_reg.has_extension('xcube.core.store', 's3'))

        self.assertTrue(ext_reg.has_extension('xcube.cli', 'compute'))
        self.assertTrue(ext_reg.has_extension('xcube.cli', 'extract'))
        self.assertTrue(ext_reg.has_extension('xcube.cli', 'gen'))
        self.assertTrue(ext_reg.has_extension('xcube.cli', 'level'))
        self.assertTrue(ext_reg.has_extension('xcube.cli', 'optimize'))



# test/webapi/helpers.py

import os
from typing import Dict, Any
from typing import Mapping, Union
from typing import Optional, Type, TypeVar

import yaml

from test.server.mocks import MockFramework
from xcube.server.api import Context
from xcube.server.api import ServerConfig
from xcube.server.framework import Framework
from xcube.server.server import Server
from xcube.server.testing import ServerTestCase
from xcube.util.extension import ExtensionRegistry
from xcube.util.plugin import get_extension_registry
from xcube.util.undefined import UNDEFINED
from xcube.webapi.context import MultiLevelDatasetOpener
from xcube.webapi.context import ServiceContext
from xcube.webapi.errors import ServiceBadRequestError
from xcube.webapi.reqparams import RequestParams

XCUBE_TEST_CLIENT_ID = os.environ.get('XCUBE_TEST_CLIENT_ID')
XCUBE_TEST_CLIENT_SECRET = os.environ.get('XCUBE_TEST_CLIENT_SECRET')

T = TypeVar('T', bound=Context)


def get_api_ctx(api_name: str,
                api_ctx_cls: Type[T],
                server_config: Union[str, ServerConfig],
                framework: Optional[Framework] = None,
                extension_registry: Optional[ExtensionRegistry] = None) -> T:
    """Get the API context object for the given
    API name *api_name*,
    API context class *api_ctx_cls*,
    and server configuration path or dictionary *server_config*.

    :param api_name: The name of the API, e.g. "auth"
    :param api_ctx_cls: The API context class
    :param server_config: Server configuration string or dictionary.
        If a relative path is passed it is made absolute using prefix
        "${project}/test/webapi/res/test".
    :param framework: Web framework, defaults to a MockFramework.
    :param extension_registry: Extension registry,
        defaults to xcube's populated default extension registry
    :return: The API context object
    :raise AssertionError: if API context object can not be determined
    """
    if isinstance(server_config, str):
        config_path = server_config
        base_dir = os.path.dirname(config_path)
        if not base_dir:
            base_dir = get_res_test_dir()
            config_path = os.path.join(base_dir, config_path)
        elif not os.path.isabs(base_dir):
            base_dir = os.path.abspath(base_dir)
        with open(config_path, encoding='utf-8') as fp:
            server_config = yaml.safe_load(fp)
            assert isinstance(server_config, dict)
            server_config["base_dir"] = base_dir
    else:
        assert isinstance(server_config, dict)

    framework = framework or MockFramework()
    extension_registry = extension_registry or get_extension_registry()
    server = Server(framework,
                    server_config,
                    extension_registry=extension_registry)
    api_ctx = server.ctx.get_api_ctx(api_name)
    assert isinstance(api_ctx, api_ctx_cls)
    return api_ctx


def new_test_service_context(config_file_name: str = 'config.yml',
                             ml_dataset_openers: Dict[
                                 str, MultiLevelDatasetOpener] = None,
                             prefix: str = None) -> ServiceContext:
    ctx = ServiceContext(base_dir=get_res_test_dir(),
                         ml_dataset_openers=ml_dataset_openers, prefix=prefix)
    config_file = os.path.join(ctx.base_dir, config_file_name)
    with open(config_file, encoding='utf-8') as fp:
        ctx.config = yaml.safe_load(fp)
    return ctx


def get_res_test_dir() -> str:
    return os.path.normpath(os.path.join(os.path.dirname(__file__), 'res', 'test'))


def get_res_demo_dir() -> str:
    return os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..', 'xcube', 'webapi', 'res', 'demo'))


class RequestParamsMock(RequestParams):
    def __init__(self, **kvp):
        self.kvp = kvp

    def get_query_arguments(self) -> Mapping[str, str]:
        return dict(self.kvp)

    def get_query_argument(self, name: str,
                           default: Optional[str] = UNDEFINED) -> Optional[
        str]:
        value = self.kvp.get(name, default)
        if value == UNDEFINED:
            raise ServiceBadRequestError(f'Missing query parameter "{name}"')
        return value


class ServerTest(ServerTestCase):

    # noinspection PyMethodMayBeStatic
    def get_config_filename(self) -> str:
        return "config.yml"

    def get_config(self) -> Dict[str, Any]:
        base_dir = get_res_test_dir()
        with open(f'{base_dir}/{self.get_config_filename()}') as fp:
            server_config = yaml.safe_load(fp)
            server_config["base_dir"] = base_dir
            return server_config

    def get_extension_registry(self):
        return get_extension_registry()


# test/webapi/test_app.py

import unittest


class AppSmokeTest(unittest.TestCase):

    def test_start_stop_service(self):
        pass
        # TODO: The following test code will cause timeouts in test/test_handlers.py - why?
        # service = new_service(args=['--port', '20001', '--update', '0'])
        # self.assertIsInstance(service, Service)
        # service.stop()
        # IOLoop.current().call_later(0.1, service.stop)
        # service.start()


# test/webapi/test_auth.py

import json
import os
import unittest
import types

import requests

from xcube.webapi.auth import AuthMixin, AuthConfig
from xcube.webapi.auth import assert_scopes
from xcube.webapi.auth import check_scopes
from xcube.webapi.errors import ServiceAuthError
from xcube.webapi.errors import ServiceConfigError


class ServiceContextMock:
    def __init__(self, config):
        self.config = config


class RequestMock:
    def __init__(self, headers):
        self.headers = headers


XCUBE_TEST_CLIENT_ID = os.environ.get('XCUBE_TEST_CLIENT_ID')
XCUBE_TEST_CLIENT_SECRET = os.environ.get('XCUBE_TEST_CLIENT_SECRET')


class AuthMixinPropsTest2(unittest.TestCase):
    def test_auth_config(self):
        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/'
            )))
        auth_config = auth_mixin.auth_config
        self.assertIsInstance(auth_config, AuthConfig)
        # Assert that it is a cached property
        self.assertIs(auth_config, auth_mixin.auth_config)

    def test_granted_scopes(self):
        # Use "permissions" claim
        self.assert_granted_scopes(
            {
                "preferred_username": "bibbi",
                "permissions": [
                    "read:dataset:*~$preferred_username/*",
                    "read:variable:*"
                ]
            },
            {
                "read:dataset:*~bibbi/*",
                "read:variable:*"
            }
        )
        # Use "scope" claim
        self.assert_granted_scopes(
            {
                "preferred_username": "bibbi",
                "scope":
                    "read:dataset:*~$preferred_username/* read:variable:*"
            },
            {
                "read:dataset:*~bibbi/*",
                "read:variable:*"
            }
        )
        # Special case where "preferred_username" claim
        # is named just "username"
        self.assert_granted_scopes(
            {
                "preferred_username": "bibbi",
                "permissions": [
                    "read:dataset:*~${username}/*",
                    "read:variable:*"
                ]
            },
            {
                "read:dataset:*~bibbi/*",
                "read:variable:*"
            }
        )

    def assert_granted_scopes(self, id_token, expected_permissions):
        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/'
            )))
        auth_mixin.service_context.must_authenticate = True

        # noinspection PyUnusedLocal,PyShadowingNames
        def _get_id_token(self, require_auth=False):
            return id_token

        auth_mixin.get_id_token = types.MethodType(_get_id_token, auth_mixin)

        self.assertEqual(expected_permissions,
                         auth_mixin.granted_scopes)


class AuthMixinIdTokenTest(unittest.TestCase):

    @unittest.skipUnless(
        XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET,
        'XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET must be set'
    )
    def test_ok(self):
        access_token = self._fetch_access_token()

        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/'
            )))
        auth_mixin.request = RequestMock(
            headers={'Authorization': f'Bearer {access_token}'}
        )

        id_token = auth_mixin.get_id_token()
        self.assertEqual('https://xcube-dev.eu.auth0.com/',
                         id_token.get('iss'))
        self.assertEqual('https://xcube-dev/api/',
                         id_token.get('aud'))
        self.assertEqual('w2NoDEryIpNRtAQVH1ToU6XTurE549FW',
                         id_token.get('azp'))
        self.assertEqual('client-credentials',
                         id_token.get('gty'))
        self.assertEqual('w2NoDEryIpNRtAQVH1ToU6XTurE549FW@clients',
                         id_token.get('sub'))
        self.assertEqual(['read:datasets'],
                         id_token.get('permissions'))
        exp = id_token.get('exp')
        iat = id_token.get('iat')
        self.assertIsInstance(exp, int)
        self.assertIsInstance(iat, int)
        self.assertEqual(86400, exp - iat)

    def _fetch_access_token(self):
        response = requests.post(
            'https://xcube-dev.eu.auth0.com/oauth/token',
            headers={'Content-Type': 'application/json'},
            json={
                "client_id": os.environ.get('XCUBE_TEST_CLIENT_ID'),
                "client_secret": os.environ.get('XCUBE_TEST_CLIENT_SECRET'),
                "audience": "https://xcube-dev/api/",
                "grant_type": "client_credentials"
            }
        )
        token_data = json.loads(response.content)
        if 'access_token' not in token_data:
            self.fail('failed to fetch access token for testing')
        access_token = token_data['access_token']
        return access_token

    def test_missing_auth_config(self):
        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config={})
        auth_mixin.request = RequestMock(
            headers={'Authorization': f'Bearer my_t0k3n'}
        )

        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_id_token(require_auth=True)

        self.assertEqual('HTTP 401: Invalid header (Received access token, '
                         'but this server doesn\'t support authentication.)',
                         f'{cm.exception}')

    def test_missing_access_token(self):
        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/'
            )))
        auth_mixin.request = RequestMock(headers={})
        self.assertEqual(None, auth_mixin.get_id_token())

    @unittest.skipUnless(
        XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET,
        'XCUBE_TEST_CLIENT_ID and XCUBE_TEST_CLIENT_SECRET must be set'
    )
    def test_expired_access_token(self):
        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/'
            )))

        expired_token = \
            "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6" \
            "Ik5VSkdSVUV5UWpNeE16UTVRVGMzTnpRM05URkNRa1ky" \
            "TURGRE5qQkJSak5ETlRBeVFrWXdOUSJ9.eyJpc3MiOiJ" \
            "odHRwczovL3hjdWJlLWRldi5ldS5hdXRoMC5jb20vIiw" \
            "ic3ViIjoidzJOb0RFcnlJcE5SdEFRVkgxVG9VNlhUdXJ" \
            "FNTQ5RldAY2xpZW50cyIsImF1ZCI6Imh0dHBzOi8veGN" \
            "1YmUtZGV2L2FwaS8iLCJpYXQiOjE1Nzk0NTUxMDMsImV" \
            "4cCI6MTU3OTU0MTUwMywiYXpwIjoidzJOb0RFcnlJcE5" \
            "SdEFRVkgxVG9VNlhUdXJFNTQ5RlciLCJndHkiOiJjbGl" \
            "lbnQtY3JlZGVudGlhbHMiLCJwZXJtaXNzaW9ucyI6W11" \
            "9.NtGuVp0yY8foR_eFiEmH0eXvCx85cvb5b1cPWCKs6L" \
            "CNniUJJ3VnX5Fek08puW7Jidy-tj1UTzyG569dSHGDu3" \
            "10Mf7xpQ9gyfZCWcaohERxsv9MrxHziqfGVnxv051rOB" \
            "_c-fYyymrFnlsIqWeahcS7znvPoMovPO7E8MxVaIqxd_" \
            "S4zKVlH025F-bDvytuwXD-rFmYVElCg7u2uOZKqjpF3l" \
            "ZCWc50_F1jSGcEQZv4daQJY-3lfU6TnEQAuGWlOVRrCN" \
            "u05nlUBFPz6G82tB_nsP1uTa8uElOzoalVttXufLIeU0" \
            "FL8Sv-lC6wUJTZAFpykLNmpA-vhkSeTqMv4g"
        auth_mixin.request = RequestMock(
            headers={'Authorization': f'Bearer {expired_token}'})
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_id_token()
        self.assertEqual('HTTP 401: Token expired (Token is expired)',
                         f'{cm.exception}')


class AuthMixinAccessTokenTest(unittest.TestCase):

    def test_ok(self):
        auth_mixin = AuthMixin()
        auth_mixin.request = RequestMock(
            headers={'Authorization': 'Bearer my_t0k3n'}
        )
        self.assertEqual('my_t0k3n', auth_mixin.get_access_token())

    def test_ok_no_auth(self):
        auth_mixin = AuthMixin()
        auth_mixin.request = RequestMock(headers={})
        self.assertEqual(None, auth_mixin.get_access_token())

    def test_not_ok(self):
        auth_mixin = AuthMixin()

        auth_mixin.request = RequestMock(headers={})
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_access_token(require_auth=True)
        self.assertEqual('HTTP 401: Authorization header'
                         ' missing (Authorization header is expected)',
                         f'{cm.exception}')

        auth_mixin.request = RequestMock(
            headers={'Authorization': 'Beerer my_t0k3n'}
        )
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_access_token()
        self.assertEqual('HTTP 401: Invalid header'
                         ' (Authorization header must start with "Bearer")',
                         f'{cm.exception}')

        auth_mixin.request = RequestMock(
            headers={'Authorization': 'Bearer'}
        )
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_access_token()
        self.assertEqual('HTTP 401: Invalid header (Bearer token not found)',
                         f'{cm.exception}')

        auth_mixin.request = RequestMock(
            headers={'Authorization': 'Bearer my t0k3n'}
        )
        with self.assertRaises(ServiceAuthError) as cm:
            auth_mixin.get_access_token()
        self.assertEqual('HTTP 401: Invalid header'
                         ' (Authorization header must be Bearer token)',
                         f'{cm.exception}')


class AuthMixinAuthConfigTest(unittest.TestCase):

    def test_ok(self):
        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Authority='https://xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/'
            )))
        auth_config = auth_mixin.auth_config
        self.assertEqual('https://xcube-dev/api/',
                         auth_config.audience)
        self.assertEqual('https://xcube-dev.eu.auth0.com',
                         auth_config.authority)
        self.assertEqual('https://xcube-dev.eu.auth0.com/'
                         '.well-known/openid-configuration',
                         auth_config.well_known_oid_config)
        self.assertEqual('https://xcube-dev.eu.auth0.com/'
                         '.well-known/jwks.json',
                         auth_config.well_known_jwks)
        self.assertEqual(['RS256'], auth_config.algorithms)

    def test_not_ok(self):
        # class ServiceContextMock:
        #     def __init__(self, config):
        #         self.config = config

        auth_mixin = AuthMixin()
        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Authority='https://xcube-dev.eu.auth0.com',
            )))

        with self.assertRaises(ServiceConfigError) as cm:
            # noinspection PyUnusedLocal
            auth_config = auth_mixin.auth_config
        self.assertEqual('HTTP 500: Missing key'
                         ' "Audience" in section "Authentication"',
                         f'{cm.exception}')

        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Audience='https://xcube-dev/api/'
            )))

        with self.assertRaises(ServiceConfigError) as cm:
            # noinspection PyUnusedLocal
            auth_config = auth_mixin.auth_config
        self.assertEqual('HTTP 500: Missing'
                         ' key "Authority" in section "Authentication"',
                         f'{cm.exception}')

        auth_mixin.service_context = ServiceContextMock(config=dict(
            Authentication=dict(
                Domain='xcube-dev.eu.auth0.com',
                Audience='https://xcube-dev/api/',
                Algorithms=[],
            )))

        with self.assertRaises(ServiceConfigError) as cm:
            # noinspection PyUnusedLocal
            auth_config = auth_mixin.auth_config
        self.assertEqual('HTTP 500: Value for key "Algorithms"'
                         ' in section "Authentication" must not be empty',
                         f'{cm.exception}')


class ScopesTest(unittest.TestCase):

    def test_check_scopes_ok(self):
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         None,
                         is_substitute=True)
        )
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test1.zarr'})
        )
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test?.zarr'})
        )
        self.assertEqual(
            True,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test1.*'})
        )

    def test_check_scopes_fails(self):
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test1.zarr'},
                         None,
                         is_substitute=False)
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test1.zarr'},
                         set(),
                         is_substitute=True)
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test1.zarr'},
                         {'read:dataset:test1.zarr'},
                         is_substitute=True)
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test2.zarr'},
                         {'read:dataset:test1.zarr'})
        )
        self.assertEqual(
            False,
            check_scopes({'read:dataset:test2.zarr'},
                         {'read:dataset:test1.zarr'},
                         is_substitute=True)
        )

    def test_assert_scopes_ok(self):
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test1.zarr'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:*'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test?.zarr'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test1.*'})
        assert_scopes({'read:dataset:test1.zarr'},
                      {'read:dataset:test2.zarr',
                       'read:dataset:test3.zarr',
                       'read:dataset:test1.zarr'})

    def test_assert_scopes_fails(self):
        with self.assertRaises(ServiceAuthError) as cm:
            assert_scopes({'read:dataset:test1.zarr'},
                          None)
        self.assertEquals(
            'HTTP 401: Missing permission'
            ' (Missing permission read:dataset:test1.zarr)',
            f'{cm.exception}'
        )

        with self.assertRaises(ServiceAuthError) as cm:
            assert_scopes({'read:dataset:test1.zarr'},
                          set())
        self.assertEquals(
            'HTTP 401: Missing permission'
            ' (Missing permission read:dataset:test1.zarr)',
            f'{cm.exception}'
        )

        with self.assertRaises(ServiceAuthError) as cm:
            assert_scopes({'read:dataset:test1.zarr'},
                          {'read:dataset:test2.zarr'})
        self.assertEquals(
            'HTTP 401: Missing permission'
            ' (Missing permission read:dataset:test1.zarr)',
            f'{cm.exception}'
        )


# test/webapi/test_config.py

import os.path
import unittest

from xcube.util.jsonschema import JsonObjectSchema
from xcube.webapi.config import DatasetConfig
from xcube.webapi.config import ServiceConfig


class ServiceConfigTest(unittest.TestCase):
    def test_get_schema(self):
        schema = ServiceConfig.get_schema()
        self.assertIsInstance(schema, JsonObjectSchema)

    def test_from_dict(self):
        test_dir = os.path.dirname(__file__)
        service_config_dict = {
            "Authentication": {
                "Authority": "https://dev.eu.auth0.com",
                "Audience": "https://xcube-dev/api/",
            },
            "DatasetAttribution":
                [
                    "© by Brockmann Consult GmbH 2020",
                    "© by EU H2020 CyanoAlert project",
                ],
            "DatasetChunkCacheSize": "100M",
            "Datasets": [
                {
                    # Will only appear for unauthorized clients
                    "Identifier": "local",
                    "Title": "Local OLCI L2C cube for region SNS",
                    "BoundingBox": [0.0, 50, 5.0, 52.5],
                    "FileSystem": "file",
                    "Path": "cube-1-250-250.zarr",
                    "Style": "default",
                    "TimeSeriesDataset": "local_ts",
                    "Augmentation": {
                        "Path": "compute_extra_vars.py",
                        "Function": "compute_variables",
                        "InputParameters": {
                            "factor_chl": 0.2,
                            "factor_tsm": 0.7,
                        },
                    },
                    "PlaceGroups": [
                        {"PlaceGroupRef": "inside-cube"},
                        {"PlaceGroupRef": "outside-cube"},
                    ],
                    "AccessControl": {
                        "IsSubstitute": True,
                    },
                }
            ],
            "DataStores": [
                {
                    "Identifier": "local",
                    "StoreId": "file",
                    "StoreParams": {
                        "root": f"{test_dir}/../../examples/serve/demo",
                    },
                    "Datasets": [
                        {
                            "Path": "*.zarr",
                            "Style": "default"
                        }
                    ]
                },
            ]
        }
        service_config = ServiceConfig.from_dict(service_config_dict)
        self.assertIsInstance(service_config, ServiceConfig)
        self.assertTrue(hasattr(service_config, 'Datasets'))
        self.assertIsInstance(service_config.Datasets, list)
        self.assertEqual(1, len(service_config.Datasets))
        self.assertIsInstance(service_config.Datasets[0], DatasetConfig)
        self.assertTrue(hasattr(service_config.Datasets[0], 'Identifier'))
        self.assertEqual("local", service_config.Datasets[0].Identifier)


# test/webapi/test_context.py

import os.path
import unittest

import xarray as xr

from test.webapi.helpers import new_test_service_context
from xcube.core.mldataset import MultiLevelDataset
from xcube.webapi.context import normalize_prefix
from xcube.webapi.errors import ServiceResourceNotFoundError
from xcube.webapi.errors import ServiceConfigError


class ServiceContextTest(unittest.TestCase):
    def test_get_dataset_and_variable(self):
        ctx = new_test_service_context()

        ds = ctx.get_dataset('demo')
        self.assertIsInstance(ds, xr.Dataset)

        ml_ds = ctx.get_ml_dataset('demo')
        self.assertIsInstance(ml_ds, MultiLevelDataset)
        self.assertIs(3, ml_ds.num_levels)
        self.assertIs(ds, ml_ds.get_dataset(0))

        for var_name in ('conc_chl', 'conc_tsm'):
            for z in range(ml_ds.num_levels):
                conc_chl_z = ctx.get_variable_for_z('demo', var_name, z)
                self.assertIsInstance(conc_chl_z, xr.DataArray)
            with self.assertRaises(ServiceResourceNotFoundError) as cm:
                ctx.get_variable_for_z('demo', var_name, 3)
            self.assertEqual(404, cm.exception.status_code)
            self.assertEqual(f'Variable "{var_name}" has no z-index 3 in dataset "demo"', cm.exception.reason)

        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            ctx.get_variable_for_z('demo', 'conc_ys', 0)
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Variable "conc_ys" not found in dataset "demo"', cm.exception.reason)

        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            ctx.get_dataset('demox')
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Dataset "demox" not found', cm.exception.reason)

        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            ctx.get_dataset('demo', expected_var_names=['conc_ys'])
        self.assertEqual(404, cm.exception.status_code)
        self.assertEqual('Variable "conc_ys" not found in dataset "demo"', cm.exception.reason)

    def test_get_dataset_with_augmentation(self):
        ctx = new_test_service_context(config_file_name='config-aug.yml')

        ds = ctx.get_dataset('demo-aug')
        self.assertIsInstance(ds, xr.Dataset)

        ml_ds = ctx.get_ml_dataset('demo-aug')
        self.assertIsInstance(ml_ds, MultiLevelDataset)
        self.assertIs(3, ml_ds.num_levels)
        self.assertIs(ds, ml_ds.get_dataset(0))

        for var_name in ('conc_chl', 'conc_tsm', 'chl_tsm_sum', 'chl_category'):
            for z in range(ml_ds.num_levels):
                conc_chl_z = ctx.get_variable_for_z('demo-aug', var_name, z)
                self.assertIsInstance(conc_chl_z, xr.DataArray)

    def test_get_dataset_configs_from_stores(self):
        ctx = new_test_service_context(config_file_name='config-datastores.yml')

        dataset_configs_from_stores = ctx.get_dataset_configs_from_stores(
            ctx.get_data_store_pool()
        )
        self.assertIsNotNone(dataset_configs_from_stores)
        self.assertEqual(3, len(dataset_configs_from_stores))
        ids = [config['Identifier'] for config in dataset_configs_from_stores]
        self.assertIn('Cube 1', ids)
        self.assertIn('Cube 5', ids)
        self.assertIn('test~cube-1-250-250.levels', ids)

    def test_get_dataset_configs_with_duplicate_ids_from_stores(self):
        ctx = new_test_service_context(
            config_file_name='config-datastores-double-ids.yml'
        )

        with self.assertRaises(ServiceConfigError) as sce:
            ctx.get_dataset_configs_from_stores(ctx.get_data_store_pool())
        self.assertEqual(sce.exception.reason,
                         'User-defined identifiers can only be assigned to '
                         'datasets with non-wildcard paths.')

    def test_config_and_dataset_cache(self):
        ctx = new_test_service_context()
        self.assertNotIn('demo', ctx.dataset_cache)

        ctx.get_dataset('demo')
        self.assertIn('demo', ctx.dataset_cache)

        ctx.config = dict(Datasets=[
            dict(Identifier='demo',
                 Path="../../../../examples/serve/demo/cube.nc"),
            dict(Identifier='demo2',
                 Path="../../../../examples/serve/demo/cube.nc"),
        ])
        self.assertNotIn('demo', ctx.dataset_cache)
        self.assertNotIn('demo2', ctx.dataset_cache)

        ctx.get_dataset('demo2')
        self.assertNotIn('demo', ctx.dataset_cache)
        self.assertIn('demo2', ctx.dataset_cache)

        ctx.config = dict(Datasets=[
            dict(Identifier='demo2',
                 Path="../../../../examples/serve/demo/cube.nc"),
        ])
        self.assertNotIn('demo', ctx.dataset_cache)
        self.assertNotIn('demo2', ctx.dataset_cache)

    def test_get_s3_bucket_mapping(self):
        ctx = new_test_service_context()
        bucket_mapping = ctx.get_s3_bucket_mapping()
        self.assertEqual(['demo'],
                         list(bucket_mapping.keys()))
        path = bucket_mapping['demo']
        self.assertTrue(os.path.isabs(path))
        self.assertTrue(path.replace('\\', '/').endswith('examples/serve/demo/cube-1-250-250.zarr'))

    def test_get_color_mapping(self):
        ctx = new_test_service_context()
        cm = ctx.get_color_mapping('demo', 'conc_chl')
        self.assertEqual(('plasma', (0., 24.)), cm)
        cm = ctx.get_color_mapping('demo', 'conc_tsm')
        self.assertEqual(('PuBuGn', (0., 100.)), cm)
        cm = ctx.get_color_mapping('demo', 'kd489')
        self.assertEqual(('jet', (0., 6.)), cm)
        with self.assertRaises(ServiceResourceNotFoundError):
            ctx.get_color_mapping('demo', '_')

    def test_get_rgb_color_mapping(self):
        ctx = new_test_service_context()
        rgb_cm = ctx.get_rgb_color_mapping('demo')
        self.assertEqual(([None, None, None], [(0.0, 1.0), (0.0, 1.0), (0.0, 1.0)]), rgb_cm)
        rgb_cm = ctx.get_rgb_color_mapping('demo', norm_range=(1.0, 2.5))
        self.assertEqual(([None, None, None], [(1.0, 2.5), (1.0, 2.5), (1.0, 2.5)]), rgb_cm)
        ctx = new_test_service_context('config-rgb.yml')
        rgb_cm = ctx.get_rgb_color_mapping('demo-rgb')
        self.assertEqual((['conc_chl', 'conc_tsm', 'kd489'], [(0.0, 24.0), (0.0, 100.0), (0.0, 6.0)]), rgb_cm)

    def test_get_style(self):
        ctx = new_test_service_context()
        style = ctx.get_style('demo-1w')
        self.assertEqual({'Identifier': 'default',
                          'ColorMappings': {'conc_chl': {'ColorBar': 'plasma',
                                                         'ValueRange': [0.0, 24.0]},
                                            'conc_tsm': {'ColorBar': 'PuBuGn',
                                                         'ValueRange': [0.0, 100.0]},
                                            'kd489': {'ColorBar': 'jet', 'ValueRange': [0.0, 6.0]}}},
                         style)

    def test_get_global_place_groups(self):
        ctx = new_test_service_context()
        place_groups = ctx.get_global_place_groups("http://localhost:9090", load_features=False)
        self.assertIsInstance(place_groups, list)
        self.assertEqual(2, len(place_groups))
        for place_group in place_groups:
            self.assertIn("type", place_group)
            self.assertIn("id", place_group)
            self.assertIn("title", place_group)
            self.assertIn("propertyMapping", place_group)
            self.assertIn("sourcePaths", place_group)
            self.assertIn("sourceEncoding", place_group)
            self.assertIn("features", place_group)
        place_group = place_groups[0]
        self.assertEqual('inside-cube', place_group['id'])
        self.assertEqual('Points inside the cube', place_group['title'])
        self.assertEqual(None, place_group['features'])
        place_group = place_groups[1]
        self.assertEqual('outside-cube', place_group['id'])
        self.assertEqual('Points outside the cube', place_group['title'])
        self.assertEqual(None, place_group['features'])

        place_groups = ctx.get_global_place_groups("http://localhost:9090", load_features=True)
        self.assertIsInstance(place_groups, list)
        self.assertEqual(2, len(place_groups))
        for place_group in place_groups:
            self.assertIn("type", place_group)
            self.assertIn("id", place_group)
            self.assertIn("title", place_group)
            self.assertIn("propertyMapping", place_group)
            self.assertIn("sourcePaths", place_group)
            self.assertIn("sourceEncoding", place_group)
            self.assertIn("features", place_group)
        place_group = place_groups[0]
        self.assertEqual('inside-cube', place_group['id'])
        self.assertEqual('Points inside the cube', place_group['title'])
        self.assertIsNotNone(place_group['features'])
        place_group = place_groups[1]
        self.assertEqual('outside-cube', place_group['id'])
        self.assertEqual('Points outside the cube', place_group['title'])
        self.assertIsNotNone(place_group['features'])

    def test_get_global_place_group(self):
        ctx = new_test_service_context()
        place_group = ctx.get_global_place_group("inside-cube", "http://localhost:9090", load_features=True)
        self.assertIsInstance(place_group, dict)
        self.assertIn("type", place_group)
        self.assertEqual("FeatureCollection", place_group["type"])
        self.assertIn("features", place_group)
        self.assertIsInstance(place_group["features"], list)
        self.assertEqual(3, len(place_group["features"]))
        self.assertIs(place_group, ctx.get_global_place_group("inside-cube", "http://localhost:9090"))
        self.assertIsNot(place_group, ctx.get_global_place_group("outside-cube", "http://localhost:9090"))

        with self.assertRaises(ServiceResourceNotFoundError) as cm:
            ctx.get_global_place_group("bibo", "http://localhost:9090")
        self.assertEqual('HTTP 404: Place group "bibo" not found', f"{cm.exception}")

    def test_get_other_store_params_than_root(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'two',
                          'Title': 'Test 2',
                          'FileSystem': 's3',
                          'Anonymous': False,
                          'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                          'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                          'Region': 'eu-central-1'}
        store_params = ctx._get_other_store_params_than_root(dataset_config)
        expected_dict = \
            {'storage_options':
                 {'anon': False,
                  'client_kwargs':
                      {'endpoint_url': 'https://s3.eu-central-1.amazonaws.com',
                       'region_name': 'eu-central-1'}
                  }
             }
        self.assertIsNotNone(store_params)
        self.assertEqual(expected_dict, store_params)


class NormalizePrefixTest(unittest.TestCase):
    def test_empty(self):
        self.assertEqual('', normalize_prefix(None))
        self.assertEqual('', normalize_prefix(''))
        self.assertEqual('', normalize_prefix('/'))
        self.assertEqual('', normalize_prefix('///'))

    def test_normalizes_slashes(self):
        self.assertEqual('/xcube/api/v1', normalize_prefix('/xcube/api/v1'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('xcube/api/v1'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('xcube/api/v1/'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('/xcube/api/v1/'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('/xcube/api/v1///'))
        self.assertEqual('/xcube/api/v1', normalize_prefix('////xcube/api/v1'))

    def test_interpolates_vars(self):
        from xcube.version import version
        self.assertEqual(f'/{version}',
                         normalize_prefix('${version}'))
        self.assertEqual(f'/xcube',
                         normalize_prefix('/${name}'))
        self.assertEqual(f'/xcube/v{version}',
                         normalize_prefix('/${name}/v${version}'))


class MaybeAssignStoreInstanceIdsTest(unittest.TestCase):

    def test_find_common_store(self):
        ctx = new_test_service_context()
        dataset_configs = [
            {
                'Identifier': 'z_0',
                'FileSystem': 'file',
                'Path': '/one/path/abc.zarr'
            },
            {
                'Identifier': 'z_1',
                'FileSystem': 'file',
                'Path': '/one/path/def.zarr'
            },
            {
                'Identifier': 'z_4',
                'FileSystem': 's3',
                'Path': '/one/path/mno.zarr'
            },
            {
                'Identifier': 'z_2',
                'FileSystem': 'file',
                'Path': '/another/path/ghi.zarr'
            },
            {
                'Identifier': 'z_3',
                'FileSystem': 'file',
                'Path': '/one/more/path/jkl.zarr'
            },
            {
                'Identifier': 'z_5',
                'FileSystem': 's3',
                'Path': '/one/path/pqr.zarr'
            },
            {
                'Identifier': 'z_6',
                'FileSystem': 'file',
                'Path': '/one/path/stu.zarr'
            },
            {
                'Identifier': 'z_7',
                'FileSystem': 'file',
                'Path': '/one/more/path/vwx.zarr'
            },
        ]
        ctx.config['Datasets'] = dataset_configs
        adjusted_dataset_configs = ctx.get_dataset_configs()

        expected_dataset_configs = [
            {
                'Identifier': 'z_0',
                'FileSystem': 'file',
                'Path': 'path/abc.zarr',
                'StoreInstanceId': 'file_2'
            },
            {
                'Identifier': 'z_1',
                'FileSystem': 'file',
                'Path': 'path/def.zarr',
                'StoreInstanceId': 'file_2'
            },
            {
                'Identifier': 'z_4',
                'FileSystem': 's3',
                'Path': 'mno.zarr',
                'StoreInstanceId': 's3_1'
            },
            {
                'Identifier': 'z_2',
                'FileSystem': 'file',
                'Path': 'ghi.zarr',
                'StoreInstanceId': 'file_1'
            },
            {
                'Identifier': 'z_3',
                'FileSystem': 'file',
                'Path': 'more/path/jkl.zarr',
                'StoreInstanceId': 'file_2'
            },
            {
                'Identifier': 'z_5',
                'FileSystem': 's3',
                'Path': 'pqr.zarr',
                'StoreInstanceId': 's3_1'
            },
            {
                'Identifier': 'z_6',
                'FileSystem': 'file',
                'Path': 'path/stu.zarr',
                'StoreInstanceId': 'file_2'
            },
            {
                'Identifier': 'z_7',
                'FileSystem': 'file',
                'Path': 'more/path/vwx.zarr',
                'StoreInstanceId': 'file_2'
            },
        ]
        self.assertEqual(expected_dataset_configs, adjusted_dataset_configs)

    def test_with_instance_id(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'zero',
                          'Title': 'Test 0',
                          'FileSystem': 'file',
                          'StoreInstanceId': 'some_id'}
        dataset_config_copy = dataset_config.copy()

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(dataset_config_copy, dataset_config)

    def test_file(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'one',
                          'Title': 'Test 1',
                          'FileSystem': 'file',
                          'Path': 'cube-1-250-250.zarr'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Path',
                         'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('one',
                         dataset_config['Identifier'])
        self.assertEqual('Test 1', dataset_config['Title'])
        self.assertEqual('file', dataset_config['FileSystem'])
        self.assertEqual('cube-1-250-250.zarr', dataset_config["Path"])
        self.assertEqual('file_1', dataset_config['StoreInstanceId'])

    def test_local(self):
        # this test tests backwards compatibility.
        # TODO please remove when support for file systems 'local' and 'obs'
        # has ended
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'one',
                          'Title': 'Test 1',
                          'FileSystem': 'local',
                          'Path': 'cube-1-250-250.zarr'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Path',
                         'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('one',
                         dataset_config['Identifier'])
        self.assertEqual('Test 1', dataset_config['Title'])
        self.assertEqual('local', dataset_config['FileSystem'])
        self.assertEqual('cube-1-250-250.zarr', dataset_config["Path"])
        self.assertEqual('file_1', dataset_config['StoreInstanceId'])

    def test_s3(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'two',
                          'Title': 'Test 2',
                          'FileSystem': 's3',
                          'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                          'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                          'Region': 'eu-central-1'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Endpoint',
                          'Path', 'Region', 'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('two', dataset_config['Identifier'])
        self.assertEqual('Test 2', dataset_config['Title'])
        self.assertEqual('s3', dataset_config['FileSystem'])
        self.assertEqual('https://s3.eu-central-1.amazonaws.com',
                         dataset_config['Endpoint'])
        self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path'])
        self.assertEqual('eu-central-1', dataset_config['Region'])
        self.assertEqual('s3_1', dataset_config['StoreInstanceId'])

    def test_obs(self):
        # this test tests backwards compatibility.
        # TODO please remove when support for file systems 'local' and 'obs'
        # has ended
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'two',
                          'Title': 'Test 2',
                          'FileSystem': 'obs',
                          'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                          'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                          'Region': 'eu-central-1'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Endpoint',
                          'Path', 'Region', 'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('two', dataset_config['Identifier'])
        self.assertEqual('Test 2', dataset_config['Title'])
        self.assertEqual('obs', dataset_config['FileSystem'])
        self.assertEqual('https://s3.eu-central-1.amazonaws.com',
                         dataset_config['Endpoint'])
        self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path'])
        self.assertEqual('eu-central-1', dataset_config['Region'])
        self.assertEqual('s3_1', dataset_config['StoreInstanceId'])

    def test_memory(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'three',
                          'Title': 'Test 3',
                          'FileSystem': 'memory'}
        dataset_config_copy = dataset_config.copy()

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(dataset_config_copy, dataset_config)

    def test_missing_file_system(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'five',
                          'Title': 'Test 5',
                          'Path': 'cube-1-250-250.zarr'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'Path', 'StoreInstanceId'],
                         list(dataset_config.keys()))
        self.assertEqual('five', dataset_config['Identifier'])
        self.assertEqual('Test 5', dataset_config['Title'])
        self.assertEqual('cube-1-250-250.zarr', dataset_config['Path'])
        self.assertEqual('file_1', dataset_config['StoreInstanceId'])

    def test_invalid_file_system(self):
        ctx = new_test_service_context()
        dataset_config = {'Identifier': 'five',
                          'Title': 'Test 5a',
                          'FileSystem': 'invalid',
                          'Path': 'cube-1-250-250.zarr'}

        ctx.config['Datasets'] = [dataset_config]
        dataset_config = ctx.get_dataset_configs()[0]

        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Path'],
                         list(dataset_config.keys()))
        self.assertEqual('five', dataset_config['Identifier'])
        self.assertEqual('Test 5a', dataset_config['Title'])
        self.assertEqual('invalid', dataset_config['FileSystem'])
        self.assertEqual('cube-1-250-250.zarr', dataset_config['Path'])

    def test_local_store_already_existing(self):
        ctx = new_test_service_context()
        dataset_config_1 = {'Identifier': 'six',
                            'Title': 'Test 6',
                            'FileSystem': 'file',
                            'Path': 'cube-1-250-250.zarr'}
        dataset_config_2 = {'Identifier': 'six_a',
                            'Title': 'Test 6 a',
                            'FileSystem': 'file',
                            'Path': 'cube-5-100-200.zarr'}

        ctx.config['Datasets'] = [dataset_config_1, dataset_config_2]
        dataset_configs = ctx.get_dataset_configs()

        self.assertEqual(dataset_configs[0]['StoreInstanceId'],
                         dataset_configs[1]['StoreInstanceId'])

    def test_s3_store_already_existing(self):
        ctx = new_test_service_context()
        dataset_config_1 = {'Identifier': 'seven',
                            'Title': 'Test 7',
                            'FileSystem': 's3',
                            'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                            'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
                            'Region': 'eu-central-1'}

        dataset_config_2 = {'Identifier': 'seven_a',
                            'Title': 'Test 7 a',
                            'FileSystem': 's3',
                            'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
                            'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-3.zarr',
                            'Region': 'eu-central-1'}

        ctx.config['Datasets'] = [dataset_config_1, dataset_config_2]
        dataset_configs = ctx.get_dataset_configs()

        self.assertEqual(dataset_configs[0]['StoreInstanceId'],
                         dataset_configs[1]['StoreInstanceId'])


# test/webapi/test_errors.py

import unittest

from tornado.web import HTTPError

from xcube.webapi.errors import ServiceError, ServiceConfigError, ServiceBadRequestError, ServiceResourceNotFoundError


class ErrorsTest(unittest.TestCase):
    def test_same_base_type(self):

        self.assertIsInstance(ServiceError(''), HTTPError)
        self.assertEqual(500, ServiceError('').status_code)
        self.assertEqual(503, ServiceError('', status_code=503).status_code)

        self.assertIsInstance(ServiceConfigError(''), ServiceError)
        self.assertEqual(500, ServiceConfigError('').status_code)

        self.assertIsInstance(ServiceBadRequestError(''), ServiceError)
        self.assertEqual(400, ServiceBadRequestError('').status_code)

        self.assertIsInstance(ServiceResourceNotFoundError(''), ServiceError)
        self.assertEqual(404, ServiceResourceNotFoundError('').status_code)


# test/webapi/test_handlers.py

from tornado.escape import json_decode
from tornado.testing import AsyncHTTPTestCase

from xcube.webapi.app import new_application
from .helpers import new_test_service_context

CTX = new_test_service_context()

# Preload datasets, so we don't run into timeout during testing

_ds = CTX.get_dataset("demo")
for name, var in _ds.coords.items():
    values = var.values

_ds = CTX.get_dataset("demo-1w")
for name, var in _ds.coords.items():
    values = var.values


# For usage of the tornado.testing.AsyncHTTPTestCase see http://www.tornadoweb.org/en/stable/testing.html


class HandlersTest(AsyncHTTPTestCase):

    def get_app(self):
        application = new_application()
        application.service_context = CTX
        return application

    def assertResponseOK(self, response):
        self.assertEqual(200, response.code, response.reason)
        self.assertEqual("OK", response.reason)

    def assertBadRequestResponse(self, response,
                                 expected_reason="Bad Request"):
        self.assertEqual(400, response.code)
        self.assertEqual(expected_reason, response.reason)

    def assertResourceNotFoundResponse(self, response,
                                       expected_reason="Not Found"):
        self.assertEqual(404, response.code)
        self.assertEqual(expected_reason, response.reason)

    def test_fetch_base(self):
        response = self.fetch(self.prefix + '/')
        self.assertResponseOK(response)

    def test_fetch_maintenance_fail(self):
        response = self.fetch(self.prefix + '/maintenance/fail')
        self.assertEqual(500, response.code)
        self.assertEqual("Internal Server Error", response.reason)
        body = json_decode(response.body)
        self.assertIn("error", body)
        self.assertEqual(500, body["error"].get("code"))
        self.assertEqual("Internal Server Error", body["error"].get("message"))

        response = self.fetch(self.prefix + '/maintenance/fail?code=403')
        self.assertEqual(403, response.code)
        self.assertEqual("Forbidden", response.reason)
        body = json_decode(response.body)
        self.assertIn("error", body)
        self.assertEqual(403, body["error"].get("code"))
        self.assertEqual("Forbidden", body["error"].get("message"))

        response = self.fetch(
            self.prefix + '/maintenance/fail?code=400&message=wrong!'
        )
        self.assertEqual(400, response.code)
        self.assertEqual("Bad Request", response.reason)
        body = json_decode(response.body)
        self.assertIn("error", body)
        self.assertEqual(400, body["error"].get("code"))
        self.assertEqual("Bad Request", body["error"].get("message"))

    def test_fetch_wmts_kvp_capabilities(self):
        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?SERVICE=WMTS'
                                            '&VERSION=1.0.0'
                                            '&REQUEST=GetCapabilities')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?service=WMTS'
                                            '&version=1.0.0'
                                            '&request=GetCapabilities')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetCapabilities')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?VERSION=1.0.0&REQUEST=GetCapabilities')
        self.assertBadRequestResponse(response,
                                      'Missing query parameter "service"')

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?SERVICE=WMS'
                                            'VERSION=1.0.0'
                                            '&REQUEST=GetCapabilities')
        self.assertBadRequestResponse(response,
                                      'Value for "service" parameter must be "WMTS"')

    def test_fetch_wmts_kvp_tile(self):
        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetTile'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=WorldCRS84Quad'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0')
        self.assertResponseOK(response)

        # issue #132 by Dirk
        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetTile'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=WorldWebMercatorQuad'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0'
                                            '&Time=2017-01-25T09%3A35%3A50')
        self.assertResponseOK(response)

        # issue #132 by Dirk
        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetTile'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=WorldWebMercatorQuad'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0'
                                            '&Time=2017-01-25T09%3A35%3A50%2F2017-01-25T10%3A20%3A15')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetTile'
                                            '&Format=image/jpg'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=WorldCRS84Quad'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0')
        self.assertBadRequestResponse(response, 'Value for "format" parameter must be "image/png"')

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.1.0'
                                            '&Request=GetTile'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=WorldCRS84Quad'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0')
        self.assertBadRequestResponse(response, 'Value for "version" parameter must be "1.0.0"')

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Request=GetTile'
                                            '&Version=1.0.0'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=conc_chl'
                                            '&TileMatrixSet=WorldCRS84Quad'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0')
        self.assertBadRequestResponse(response,
                                      'Value for "layer" parameter must be "<dataset>.<variable>"')

        response = self.fetch(self.prefix + '/wmts/kvp'
                                            '?Service=WMTS'
                                            '&Version=1.0.0'
                                            '&Request=GetTile'
                                            '&Format=image/png'
                                            '&Style=Default'
                                            '&Layer=demo.conc_chl'
                                            '&TileMatrixSet=TileGrid_2000_1000'
                                            '&TileMatrix=0'
                                            '&TileRow=0'
                                            '&TileCol=0')
        self.assertBadRequestResponse(
            response,
            'Value for "tilematrixset" parameter must'
            ' be one of (\'WorldCRS84Quad\', \'WorldWebMercatorQuad\')'
        )

    def test_fetch_wmts_capabilities(self):
        response = self.fetch(
            self.prefix + '/wmts/1.0.0/WMTSCapabilities.xml')
        self.assertResponseOK(response)

    def test_fetch_wmts_tile(self):
        response = self.fetch(
            self.prefix
            + '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_geo(self):
        response = self.fetch(
            self.prefix
            + '/wmts/1.0.0/tile/demo/conc_chl/WorldCRS84Quad/0/0/0.png'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_mercator(self):
        response = self.fetch(
            self.prefix
            + '/wmts/1.0.0/tile/demo/conc_chl/WorldWebMercatorQuad/0/0/0.png'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_with_params(self):
        response = self.fetch(
            self.prefix
            + '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png'
              '?time=current&cbar=jet'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_with_params_geo(self):
        response = self.fetch(
            self.prefix
            + '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png'
              '?time=current&cbar=jet&TileMatrixSet=WorldCRS84Quad'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_with_params_mercator(self):
        response = self.fetch(
            self.prefix
            + '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png'
              '?time=current&cbar=jet&TileMatrixSet=WorldWebMercatorQuad'
        )
        self.assertResponseOK(response)

    def test_fetch_datasets(self):
        response = self.fetch(self.prefix + '/datasets')
        self.assertResponseOK(response)

    def test_fetch_datasets_details(self):
        response = self.fetch(self.prefix + '/datasets?details=1')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/datasets?details=1&tiles=ol')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/datasets?details=1&point=2.8,51.0')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/datasets?details=1&point=2,8a,51.0')
        self.assertBadRequestResponse(response,
                                      "Parameter 'point' parameter must be a point using format"
                                      " '<lon>,<lat>', but was '2,8a,51.0'")

    def test_fetch_dataset(self):
        response = self.fetch(self.prefix + '/datasets/demo')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/datasets/demo?tiles=ol4')
        self.assertResponseOK(response)

    def test_fetch_dataset_coords(self):
        response = self.fetch(self.prefix + '/datasets/demo/coords/time')
        self.assertResponseOK(response)

    def test_fetch_list_s3bucket(self):
        response = self.fetch(self.prefix + '/s3bucket')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket?delimiter=%2F')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket?delimiter=%2F&prefix=demo%2F')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket?delimiter=%2F&list-type=2')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket?delimiter=%2F&prefix=demo%2F&list-type=2')
        self.assertResponseOK(response)

    def test_fetch_head_s3bucket_object(self):
        self._assert_fetch_head_s3bucket_object(method='HEAD')

    def test_fetch_get_s3bucket_object(self):
        self._assert_fetch_head_s3bucket_object(method='GET')

    def _assert_fetch_head_s3bucket_object(self, method):
        response = self.fetch(self.prefix + '/s3bucket/demo', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/.zattrs', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/.zgroup', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/.zarray', method=method)
        self.assertResourceNotFoundResponse(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/time/.zattrs', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/time/.zarray', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/time/.zgroup', method=method)
        self.assertResourceNotFoundResponse(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/time/0', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/.zattrs', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/.zarray', method=method)
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/.zgroup', method=method)
        self.assertResourceNotFoundResponse(response)
        response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/3.2.4', method=method)
        self.assertResponseOK(response)

    def test_fetch_dataset_tile(self):
        response = self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tiles/0/0/0.png')
        self.assertResponseOK(response)

    def test_fetch_dataset_rgb_tile(self):
        response = self.fetch(self.prefix + '/datasets/demo/vars/rgb/tiles/0/0/0.png?'
                                            'r=kd489&g=conc_chl&b=conc_tsm&time=current')
        self.assertResponseOK(response)

    def test_fetch_dataset_tile_with_params(self):
        response = self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tiles/0/0/0.png?'
                                            'time=current&cbar=jet&debug=1')
        self.assertResponseOK(response)

    def test_fetch_dataset_tile_grid_ol4_json(self):
        response = self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tilegrid?tiles=ol4')
        self.assertResponseOK(response)

    def test_fetch_dataset_tile_grid_cesium_json(self):
        self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tilegrid?tiles=cesium')

    def test_fetch_color_bars_json(self):
        response = self.fetch(self.prefix + '/colorbars')
        self.assertResponseOK(response)

    def test_fetch_color_bars_html(self):
        response = self.fetch(self.prefix + '/colorbars.html')
        self.assertResponseOK(response)

    def test_fetch_all_places(self):
        response = self.fetch(self.prefix + '/places')
        self.assertResponseOK(response)

    def test_fetch_places(self):
        response = self.fetch(self.prefix + '/places/all')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/places/all?bbox=10,10,20,20')
        self.assertResponseOK(response)

    def test_fetch_dataset_places(self):
        response = self.fetch(self.prefix + '/datasets/demo/places')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/places/all/demo')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/places/inside-cube/demo')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/places/bibo/demo')
        self.assertResourceNotFoundResponse(response, 'Place group "bibo" not found')
        response = self.fetch(self.prefix + '/places/inside-cube/bibo')
        self.assertResourceNotFoundResponse(response, 'Dataset "bibo" not found')

    def test_fetch_timeseries_invalid_body(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='')
        self.assertBadRequestResponse(response, 'Invalid or missing GeoJSON object in request body')

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type":"Point"}')
        self.assertBadRequestResponse(response, 'GeoJSON object expected')

    def test_fetch_timeseries_geometry(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "Point", "coordinates": [1, 51]}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type":"Polygon", "coordinates": [[[1, 51], [2, 51], [2, 52], [1, 51]]]}')
        self.assertResponseOK(response)

    def test_fetch_timeseries_geometry_collection(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "GeometryCollection", "geometries": null}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "GeometryCollection", "geometries": []}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "GeometryCollection", "geometries": '
                                   '[{"type": "Point", "coordinates": [1, 51]}]}')
        self.assertResponseOK(response)

    def test_fetch_timeseries_feature(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "Feature", '
                                   ' "properties": {}, '
                                   ' "geometry": {"type": "Point", "coordinates": [1, 51]}'
                                   '}')
        self.assertResponseOK(response)

    def test_fetch_timeseries_feature_collection(self):
        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "FeatureCollection", "features": null}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "FeatureCollection", "features": []}')
        self.assertResponseOK(response)

        response = self.fetch(self.prefix + '/timeseries/demo/conc_chl', method="POST",
                              body='{"type": "FeatureCollection", "features": ['
                                   '  {"type": "Feature", "properties": {}, '
                                   '   "geometry": {"type": "Point", "coordinates": [1, 51]}}'
                                   ']}')
        self.assertResponseOK(response)

    def test_fetch_ts_legacy_info(self):
        response = self.fetch(self.prefix + '/ts')
        self.assertResponseOK(response)

    def test_fetch_ts_legacy_point(self):
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point')
        self.assertBadRequestResponse(response, 'Missing query parameter "lon"')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point?lon=2.1')
        self.assertBadRequestResponse(response, 'Missing query parameter "lat"')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point?lon=2.1&lat=51.1&maxValids=-5')
        self.assertBadRequestResponse(response, 'If given, query parameter "maxValids" must be -1 or positive')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point?lon=120.5&lat=-12.4')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/point?lon=2.1&lat=51.1')
        self.assertResponseOK(response)

    def test_fetch_ts_legacy_geometry(self):
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometry', method="POST",
                              body='')
        self.assertBadRequestResponse(response, 'Invalid or missing GeoJSON geometry in request body')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometry', method="POST",
                              body='{"type":"Point"}')
        self.assertBadRequestResponse(response, 'Invalid GeoJSON geometry')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometry', method="POST",
                              body='{"type": "Point", "coordinates": [1, 51]}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometry', method="POST",
                              body='{"type":"Polygon", "coordinates": [[[1, 51], [2, 51], [2, 52], [1, 51]]]}')

        self.assertResponseOK(response)

    def test_fetch_ts_legacy_geometries(self):
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='')
        self.assertBadRequestResponse(response, 'Invalid or missing GeoJSON geometry collection in request body')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='{"type":"Point"}')
        self.assertBadRequestResponse(response, 'Invalid GeoJSON geometry collection')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='{"type": "GeometryCollection", "geometries": null}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='{"type": "GeometryCollection", "geometries": []}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/geometries', method="POST",
                              body='{"type": "GeometryCollection", "geometries": '
                                   '[{"type": "Point", "coordinates": [1, 51]}]}')
        self.assertResponseOK(response)

    def test_fetch_ts_legacy_features(self):
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='')
        self.assertBadRequestResponse(response, 'Invalid or missing GeoJSON feature collection in request body')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='{"type":"Point"}')
        self.assertBadRequestResponse(response, 'Invalid GeoJSON feature collection')
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='{"type": "FeatureCollection", "features": null}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='{"type": "FeatureCollection", "features": []}')
        self.assertResponseOK(response)
        response = self.fetch(self.prefix + '/ts/demo/conc_chl/features', method="POST",
                              body='{"type": "FeatureCollection", "features": ['
                                   '  {"type": "Feature", "properties": {}, '
                                   '   "geometry": {"type": "Point", "coordinates": [1, 51]}}'
                                   ']}')
        self.assertResponseOK(response)

    @property
    def prefix(self):
        return ''


# test/webapi/test_reqparams.py

import unittest

from test.webapi.helpers import RequestParamsMock
from xcube.webapi.errors import ServiceBadRequestError
from xcube.webapi.reqparams import RequestParams


class RequestParamsTest(unittest.TestCase):
    def test_to_int(self):
        self.assertEqual(2123, RequestParams.to_int('x', '2123'))
        with self.assertRaises(ServiceBadRequestError):
            RequestParams.to_int('x', None)
        with self.assertRaises(ServiceBadRequestError):
            RequestParams.to_int('x', 'bibo')

    def test_to_float(self):
        self.assertEqual(-0.2, RequestParams.to_float('x', '-0.2'))
        with self.assertRaises(ServiceBadRequestError):
            RequestParams.to_float('x', None)
        with self.assertRaises(ServiceBadRequestError):
            RequestParams.to_float('x', 'bibo')

    def test_get_query_argument(self):
        rp = RequestParamsMock()
        self.assertEqual('bert', rp.get_query_argument('s', 'bert'))
        self.assertEqual(234, rp.get_query_argument_int('i', 234))
        self.assertEqual(0.2, rp.get_query_argument_float('f', 0.2))
        self.assertEqual((-10.4, 51.8), rp.get_query_argument_point('p', (-10.4, 51.8)))

        rp = RequestParamsMock(s='bibo', i='465', f='0.1', p='-10.4, 51.8')
        self.assertEqual('bibo', rp.get_query_argument('s', None))
        self.assertEqual(465, rp.get_query_argument_int('i', None))
        self.assertEqual(465., rp.get_query_argument_float('i', None))
        self.assertEqual(0.1, rp.get_query_argument_float('f', None))
        self.assertEqual((-10.4, 51.8), rp.get_query_argument_point('p', None))


# test/webapi/test_s3buckethandlers.py

import unittest
import urllib.request

import numpy as np
import xarray as xr

from xcube.core.dsio import open_cube

SKIP_HELP = ('Skipped, because server is not running:'
             ' $ xcube serve --verbose -c examples/serve/demo/config.yml')
SERVER_URL = 'http://localhost:8080'
ENDPOINT_URL = SERVER_URL + '/s3bucket'


def is_server_running() -> bool:
    # noinspection PyBroadException
    try:
        with urllib.request.urlopen(SERVER_URL, timeout=2.0) as response:
            response.read()
    except Exception:
        return False
    return 200 <= response.code < 400


XCUBE_SERVER_IS_RUNNING = is_server_running()


class S3BucketHandlersTest(unittest.TestCase):

    @unittest.skipUnless(XCUBE_SERVER_IS_RUNNING, SKIP_HELP)
    def test_open_cube_from_xube_server_rel_path(self):
        ds = open_cube('s3bucket/local',
                       format_name='zarr',
                       s3_kwargs={
                           'anon': True
                       },
                       s3_client_kwargs=dict(endpoint_url=SERVER_URL))
        self.assertCubeOk(ds)

    @unittest.skipUnless(XCUBE_SERVER_IS_RUNNING, SKIP_HELP)
    def test_open_cube_from_xube_server_abs_path(self):
        ds = open_cube('http://localhost:8080/s3bucket/local',
                       format_name='zarr',
                       s3_kwargs={
                           'anon': True
                       })
        self.assertCubeOk(ds)

    def assertCubeOk(self, ds):
        self.assertIsInstance(ds, xr.Dataset)
        self.assertEqual((5, 1000, 2000), ds.conc_chl.shape)
        self.assertEqual(('time', 'lat', 'lon'), ds.conc_chl.dims)
        conc_chl_values = ds.conc_chl.values
        self.assertEqual((5, 1000, 2000), conc_chl_values.shape)
        self.assertAlmostEqual(0.00005656,
                               float(np.nanmin(conc_chl_values)),
                               delta=1e-6)
        self.assertAlmostEqual(22.4421215,
                               float(np.nanmax(conc_chl_values)),
                               delta=1e-6)


# test/webapi/test_s3util.py

import os.path
import os.path
import unittest
from abc import abstractmethod, ABCMeta
from typing import Dict

from xcube.core.dsio import rimraf
from xcube.core.dsio import write_cube
from xcube.core.new import new_cube
from xcube.webapi.s3util import list_bucket_result_to_xml, list_s3_bucket_keys, list_s3_bucket_v2, list_s3_bucket_v1

S3_BUCKET = os.path.join(os.path.dirname(__file__), "s3-bucket")
TEST_CUBE_1 = os.path.join(S3_BUCKET, "test-1.zarr")
TEST_CUBE_2 = os.path.join(S3_BUCKET, "test-2.zarr")
S3_BUCKET_DICT = {'bibo.zarr': TEST_CUBE_1,
                  'bert.zarr': TEST_CUBE_2}


class S3BucketTest(unittest.TestCase, metaclass=ABCMeta):

    def setUp(self):
        self.maxDiff = None

    @classmethod
    def setUpClass(cls) -> None:
        rimraf(S3_BUCKET)
        os.mkdir(S3_BUCKET)
        cube = new_cube(time_periods=3,
                        variables=dict(precipitation=0.9,
                                       temperature=278.3)).chunk(dict(time=1, lat=90, lon=90))
        write_cube(cube, TEST_CUBE_1, "zarr", cube_asserted=True)
        write_cube(cube, TEST_CUBE_2, "zarr", cube_asserted=True)

    @classmethod
    def tearDownClass(cls) -> None:
        rimraf(S3_BUCKET)


class ListS3BucketKeys(S3BucketTest):
    def test_list_bucket_keys(self):
        result = [key_path for key_path in list_s3_bucket_keys(S3_BUCKET_DICT)]
        self.assertEqual(EXPECTED_KEYS, [key for key, _ in result])
        self.assertEqual(172, len(EXPECTED_KEYS))


class ListS3BucketTest(S3BucketTest, metaclass=ABCMeta):

    @abstractmethod
    def list_bucket(self, bucket_dict, **kwargs):
        pass

    @abstractmethod
    def assert_list_bucket_result(self, list_bucket_result, **kwargs):
        pass


# noinspection PyUnresolvedReferences
class ListS3BucketV12TestsMixin:

    def test_list_bucket_v12(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT)
        self.assert_list_bucket_result(list_bucket_result)
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertEqual(172, len(list_bucket_result.get('Contents')))
        self.assertNotIn('CommonPrefixes', list_bucket_result)

    def test_list_bucket_v12_prefix(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, prefix='bibo.zarr/')
        self.assert_list_bucket_result(list_bucket_result, prefix='bibo.zarr/')
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertEqual(86, len(list_bucket_result.get('Contents')))
        self.assertNotIn('CommonPrefixes', list_bucket_result)

    def test_list_bucket_v12_delimiter(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/')
        self.assertNotIn('Contents', list_bucket_result)
        self.assertIsInstance(list_bucket_result.get('CommonPrefixes'), list)
        self.assertEqual([{'Prefix': 'bert.zarr/'},
                          {'Prefix': 'bibo.zarr/'}],
                         list_bucket_result.get('CommonPrefixes'))

    def test_list_bucket_v12_delimiter_prefix(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/',
                                              prefix='bert.zarr/')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/',
                                       prefix='bert.zarr/')
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertIsInstance(list_bucket_result.get('CommonPrefixes'), list)
        self.assertEqual([{'ETag': '"5fb93bce985c60d724d9869f32e02d24"',
                           'Key': 'bert.zarr/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"16a49349c15b9cd60c5d8a05fc6ad649"',
                           'Key': 'bert.zarr/.zattrs',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 426,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"0a44cbe4de5e2936112efc2ed25b6223"',
                           'Key': 'bert.zarr/.zgroup',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 24,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"cc1a0115345e38f58a88b83722a9d2d8"',
                           'Key': 'bert.zarr/.zmetadata',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 6572,
                           'StorageClass': 'STANDARD'}],
                         list_bucket_result.get('Contents'))
        self.assertEqual([{'Prefix': 'bert.zarr/lat/'},
                          {'Prefix': 'bert.zarr/lat_bnds/'},
                          {'Prefix': 'bert.zarr/lon/'},
                          {'Prefix': 'bert.zarr/lon_bnds/'},
                          {'Prefix': 'bert.zarr/precipitation/'},
                          {'Prefix': 'bert.zarr/temperature/'},
                          {'Prefix': 'bert.zarr/time/'},
                          {'Prefix': 'bert.zarr/time_bnds/'}],
                         list_bucket_result.get('CommonPrefixes'))

    def test_list_bucket_v12_delimiter_prefix_2(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/',
                                              prefix='bert.zarr')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/',
                                       prefix='bert.zarr')
        self.assertNotIn('Contents', list_bucket_result)
        self.assertIsInstance(list_bucket_result.get('CommonPrefixes'), list)
        self.assertEqual([{'Prefix': 'bert.zarr/'}],
                         list_bucket_result.get('CommonPrefixes'))


class ListBucketV1Test(ListS3BucketTest, ListS3BucketV12TestsMixin):
    def test_list_bucket_v1_truncated(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT,
                                              max_keys=5)
        self.assert_list_bucket_result(list_bucket_result, max_keys=5, is_truncated=True,
                                       next_marker='bert.zarr/lat/.zarray')
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertEqual([{'ETag': '"5fb93bce985c60d724d9869f32e02d24"',
                           'Key': 'bert.zarr/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"16a49349c15b9cd60c5d8a05fc6ad649"',
                           'Key': 'bert.zarr/.zattrs',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 426,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"0a44cbe4de5e2936112efc2ed25b6223"',
                           'Key': 'bert.zarr/.zgroup',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 24,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"cc1a0115345e38f58a88b83722a9d2d8"',
                           'Key': 'bert.zarr/.zmetadata',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 6572,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"f64109793f6ea9ec1c9cfcb8ee97e145"',
                           'Key': 'bert.zarr/lat/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'}],
                         list_bucket_result.get('Contents'))
        self.assertNotIn('CommonPrefixes', list_bucket_result)

    def test_list_bucket_v1_result_to_xml(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/',
                                              max_keys=10, prefix='bibo.zarr/')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/',
                                       max_keys=10, prefix='bibo.zarr/')
        xml = list_bucket_result_to_xml(list_bucket_result)
        with open(os.path.join(os.path.dirname(__file__),
                               'res',
                               's3',
                               'list-bucket-v1-result.xml')) as fp:
            expected_xml = fp.read()
        self.assertEqual(expected_xml, xml)

    def list_bucket(self, bucket_entries: Dict[str, str], **kwargs):
        return list_s3_bucket_v1(bucket_entries,
                                 key_to_etag=True,
                                 last_modified='2019-06-24T20:43:40.862Z',
                                 **kwargs)

    def assert_list_bucket_result(self,
                                  list_bucket_result,
                                  name="s3bucket",
                                  prefix=None,
                                  delimiter=None,
                                  max_keys=1000,
                                  is_truncated=False,
                                  marker=None,
                                  next_marker=None):
        self.assertIsInstance(list_bucket_result, dict)
        self.assertEqual(name, list_bucket_result.get('Name'))
        self.assertEqual(prefix, list_bucket_result.get('Prefix'))
        self.assertEqual(delimiter, list_bucket_result.get('Delimiter'))
        self.assertEqual(max_keys, list_bucket_result.get('MaxKeys'))
        self.assertEqual(is_truncated, list_bucket_result.get('IsTruncated'))
        self.assertEqual(marker, list_bucket_result.get('Marker'))
        self.assertEqual(next_marker, list_bucket_result.get('NextMarker'))


class ListS3BucketV2Test(ListS3BucketTest, ListS3BucketV12TestsMixin):

    def test_list_bucket_v2_truncated(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT,
                                              max_keys=5)
        self.assert_list_bucket_result(list_bucket_result, max_keys=5, is_truncated=True,
                                       next_continuation_token=6)
        self.assertIsInstance(list_bucket_result.get('Contents'), list)
        self.assertEqual([{'ETag': '"5fb93bce985c60d724d9869f32e02d24"',
                           'Key': 'bert.zarr/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"16a49349c15b9cd60c5d8a05fc6ad649"',
                           'Key': 'bert.zarr/.zattrs',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 426,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"0a44cbe4de5e2936112efc2ed25b6223"',
                           'Key': 'bert.zarr/.zgroup',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 24,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"cc1a0115345e38f58a88b83722a9d2d8"',
                           'Key': 'bert.zarr/.zmetadata',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 6572,
                           'StorageClass': 'STANDARD'},
                          {'ETag': '"f64109793f6ea9ec1c9cfcb8ee97e145"',
                           'Key': 'bert.zarr/lat/',
                           'LastModified': '2019-06-24T20:43:40.862Z',
                           'Size': 0,
                           'StorageClass': 'STANDARD'}],
                         list_bucket_result.get('Contents'))
        self.assertNotIn('CommonPrefixes', list_bucket_result)

    def test_list_bucket_v2_result_to_xml(self):
        list_bucket_result = self.list_bucket(S3_BUCKET_DICT, delimiter='/',
                                              max_keys=10, prefix='bibo.zarr/')
        self.assert_list_bucket_result(list_bucket_result, delimiter='/',
                                       max_keys=10, prefix='bibo.zarr/')
        xml = list_bucket_result_to_xml(list_bucket_result)
        with open(os.path.join(os.path.dirname(__file__),
                               'res',
                               's3',
                               'list-bucket-v2-result.xml')) as fp:
            expected_xml = fp.read()
        self.assertEqual(expected_xml, xml)

    def list_bucket(self, bucket_entries: Dict[str, str], **kwargs):
        return list_s3_bucket_v2(bucket_entries,
                                 key_to_etag=True,
                                 last_modified='2019-06-24T20:43:40.862Z',
                                 **kwargs)

    def assert_list_bucket_result(self,
                                  list_bucket_result,
                                  name="s3bucket",
                                  prefix=None,
                                  delimiter=None, max_keys=1000,
                                  is_truncated=False,
                                  start_after=None,
                                  continuation_token=None,
                                  next_continuation_token=None):
        self.assertIsInstance(list_bucket_result, dict)
        self.assertEqual(name, list_bucket_result.get('Name'))
        self.assertEqual(prefix, list_bucket_result.get('Prefix'))
        self.assertEqual(delimiter, list_bucket_result.get('Delimiter'))
        self.assertEqual(max_keys, list_bucket_result.get('MaxKeys'))
        self.assertEqual(is_truncated, list_bucket_result.get('IsTruncated'))
        self.assertEqual(start_after, list_bucket_result.get('Marker'))
        self.assertEqual(continuation_token, list_bucket_result.get('ContinuationToken'))
        self.assertEqual(next_continuation_token, list_bucket_result.get('NextContinuationToken'))


EXPECTED_KEYS = [
    'bert.zarr/',
    'bert.zarr/.zattrs',
    'bert.zarr/.zgroup',
    'bert.zarr/.zmetadata',
    'bert.zarr/lat/',
    'bert.zarr/lat/.zarray',
    'bert.zarr/lat/.zattrs',
    'bert.zarr/lat/0',
    'bert.zarr/lat_bnds/',
    'bert.zarr/lat_bnds/.zarray',
    'bert.zarr/lat_bnds/.zattrs',
    'bert.zarr/lat_bnds/0.0',
    'bert.zarr/lat_bnds/1.0',
    'bert.zarr/lon/',
    'bert.zarr/lon/.zarray',
    'bert.zarr/lon/.zattrs',
    'bert.zarr/lon/0',
    'bert.zarr/lon_bnds/',
    'bert.zarr/lon_bnds/.zarray',
    'bert.zarr/lon_bnds/.zattrs',
    'bert.zarr/lon_bnds/0.0',
    'bert.zarr/lon_bnds/1.0',
    'bert.zarr/lon_bnds/2.0',
    'bert.zarr/lon_bnds/3.0',
    'bert.zarr/precipitation/',
    'bert.zarr/precipitation/.zarray',
    'bert.zarr/precipitation/.zattrs',
    'bert.zarr/precipitation/0.0.0',
    'bert.zarr/precipitation/0.0.1',
    'bert.zarr/precipitation/0.0.2',
    'bert.zarr/precipitation/0.0.3',
    'bert.zarr/precipitation/0.1.0',
    'bert.zarr/precipitation/0.1.1',
    'bert.zarr/precipitation/0.1.2',
    'bert.zarr/precipitation/0.1.3',
    'bert.zarr/precipitation/1.0.0',
    'bert.zarr/precipitation/1.0.1',
    'bert.zarr/precipitation/1.0.2',
    'bert.zarr/precipitation/1.0.3',
    'bert.zarr/precipitation/1.1.0',
    'bert.zarr/precipitation/1.1.1',
    'bert.zarr/precipitation/1.1.2',
    'bert.zarr/precipitation/1.1.3',
    'bert.zarr/precipitation/2.0.0',
    'bert.zarr/precipitation/2.0.1',
    'bert.zarr/precipitation/2.0.2',
    'bert.zarr/precipitation/2.0.3',
    'bert.zarr/precipitation/2.1.0',
    'bert.zarr/precipitation/2.1.1',
    'bert.zarr/precipitation/2.1.2',
    'bert.zarr/precipitation/2.1.3',
    'bert.zarr/temperature/',
    'bert.zarr/temperature/.zarray',
    'bert.zarr/temperature/.zattrs',
    'bert.zarr/temperature/0.0.0',
    'bert.zarr/temperature/0.0.1',
    'bert.zarr/temperature/0.0.2',
    'bert.zarr/temperature/0.0.3',
    'bert.zarr/temperature/0.1.0',
    'bert.zarr/temperature/0.1.1',
    'bert.zarr/temperature/0.1.2',
    'bert.zarr/temperature/0.1.3',
    'bert.zarr/temperature/1.0.0',
    'bert.zarr/temperature/1.0.1',
    'bert.zarr/temperature/1.0.2',
    'bert.zarr/temperature/1.0.3',
    'bert.zarr/temperature/1.1.0',
    'bert.zarr/temperature/1.1.1',
    'bert.zarr/temperature/1.1.2',
    'bert.zarr/temperature/1.1.3',
    'bert.zarr/temperature/2.0.0',
    'bert.zarr/temperature/2.0.1',
    'bert.zarr/temperature/2.0.2',
    'bert.zarr/temperature/2.0.3',
    'bert.zarr/temperature/2.1.0',
    'bert.zarr/temperature/2.1.1',
    'bert.zarr/temperature/2.1.2',
    'bert.zarr/temperature/2.1.3',
    'bert.zarr/time/',
    'bert.zarr/time/.zarray',
    'bert.zarr/time/.zattrs',
    'bert.zarr/time/0',
    'bert.zarr/time_bnds/',
    'bert.zarr/time_bnds/.zarray',
    'bert.zarr/time_bnds/.zattrs',
    'bert.zarr/time_bnds/0.0',
    'bibo.zarr/',
    'bibo.zarr/.zattrs',
    'bibo.zarr/.zgroup',
    'bibo.zarr/.zmetadata',
    'bibo.zarr/lat/',
    'bibo.zarr/lat/.zarray',
    'bibo.zarr/lat/.zattrs',
    'bibo.zarr/lat/0',
    'bibo.zarr/lat_bnds/',
    'bibo.zarr/lat_bnds/.zarray',
    'bibo.zarr/lat_bnds/.zattrs',
    'bibo.zarr/lat_bnds/0.0',
    'bibo.zarr/lat_bnds/1.0',
    'bibo.zarr/lon/',
    'bibo.zarr/lon/.zarray',
    'bibo.zarr/lon/.zattrs',
    'bibo.zarr/lon/0',
    'bibo.zarr/lon_bnds/',
    'bibo.zarr/lon_bnds/.zarray',
    'bibo.zarr/lon_bnds/.zattrs',
    'bibo.zarr/lon_bnds/0.0',
    'bibo.zarr/lon_bnds/1.0',
    'bibo.zarr/lon_bnds/2.0',
    'bibo.zarr/lon_bnds/3.0',
    'bibo.zarr/precipitation/',
    'bibo.zarr/precipitation/.zarray',
    'bibo.zarr/precipitation/.zattrs',
    'bibo.zarr/precipitation/0.0.0',
    'bibo.zarr/precipitation/0.0.1',
    'bibo.zarr/precipitation/0.0.2',
    'bibo.zarr/precipitation/0.0.3',
    'bibo.zarr/precipitation/0.1.0',
    'bibo.zarr/precipitation/0.1.1',
    'bibo.zarr/precipitation/0.1.2',
    'bibo.zarr/precipitation/0.1.3',
    'bibo.zarr/precipitation/1.0.0',
    'bibo.zarr/precipitation/1.0.1',
    'bibo.zarr/precipitation/1.0.2',
    'bibo.zarr/precipitation/1.0.3',
    'bibo.zarr/precipitation/1.1.0',
    'bibo.zarr/precipitation/1.1.1',
    'bibo.zarr/precipitation/1.1.2',
    'bibo.zarr/precipitation/1.1.3',
    'bibo.zarr/precipitation/2.0.0',
    'bibo.zarr/precipitation/2.0.1',
    'bibo.zarr/precipitation/2.0.2',
    'bibo.zarr/precipitation/2.0.3',
    'bibo.zarr/precipitation/2.1.0',
    'bibo.zarr/precipitation/2.1.1',
    'bibo.zarr/precipitation/2.1.2',
    'bibo.zarr/precipitation/2.1.3',
    'bibo.zarr/temperature/',
    'bibo.zarr/temperature/.zarray',
    'bibo.zarr/temperature/.zattrs',
    'bibo.zarr/temperature/0.0.0',
    'bibo.zarr/temperature/0.0.1',
    'bibo.zarr/temperature/0.0.2',
    'bibo.zarr/temperature/0.0.3',
    'bibo.zarr/temperature/0.1.0',
    'bibo.zarr/temperature/0.1.1',
    'bibo.zarr/temperature/0.1.2',
    'bibo.zarr/temperature/0.1.3',
    'bibo.zarr/temperature/1.0.0',
    'bibo.zarr/temperature/1.0.1',
    'bibo.zarr/temperature/1.0.2',
    'bibo.zarr/temperature/1.0.3',
    'bibo.zarr/temperature/1.1.0',
    'bibo.zarr/temperature/1.1.1',
    'bibo.zarr/temperature/1.1.2',
    'bibo.zarr/temperature/1.1.3',
    'bibo.zarr/temperature/2.0.0',
    'bibo.zarr/temperature/2.0.1',
    'bibo.zarr/temperature/2.0.2',
    'bibo.zarr/temperature/2.0.3',
    'bibo.zarr/temperature/2.1.0',
    'bibo.zarr/temperature/2.1.1',
    'bibo.zarr/temperature/2.1.2',
    'bibo.zarr/temperature/2.1.3',
    'bibo.zarr/time/',
    'bibo.zarr/time/.zarray',
    'bibo.zarr/time/.zattrs',
    'bibo.zarr/time/0',
    'bibo.zarr/time_bnds/',
    'bibo.zarr/time_bnds/.zarray',
    'bibo.zarr/time_bnds/.zattrs',
    'bibo.zarr/time_bnds/0.0'
]


# test/webapi/test_service.py

import re
import unittest

from xcube.webapi.service import Service
from xcube.webapi.service import new_default_config
from xcube.webapi.service import url_pattern


class ServerMock:
    def stop(self):
        pass


class ApplicationMock:
    # noinspection PyMethodMayBeStatic,PyUnusedLocal
    def listen(self, port: int, address: str = "", **kwargs):
        return ServerMock()


class ServiceTest(unittest.TestCase):

    def test_service_ok(self):
        application = ApplicationMock()
        # noinspection PyTypeChecker
        service = Service(application)
        self.assertIs(application, service.application)
        self.assertTrue(hasattr(service.application, 'service_context'))
        self.assertTrue(hasattr(service.application, 'time_of_last_activity'))

    def test_service_deprecated(self):
        application = ApplicationMock()
        # noinspection PyTypeChecker
        service = Service(application,
                          log_to_stderr=True,
                          log_file_prefix='log-')
        self.assertIs(application, service.application)

    def test_service_kwarg_validation(self):
        application = ApplicationMock()

        for k, v in dict(cube_paths=['test.zarr'],
                         styles=dict(a=2),
                         aws_prof='test',
                         aws_env=True).items():
            with self.assertRaises(ValueError) as cm:
                # noinspection PyTypeChecker
                Service(application,
                        config_file='server-conf.yaml',
                        **{k: v})
            self.assertEqual(f'config_file and {k} cannot be given both',
                             f'{cm.exception}')

    def test_start_and_stop(self):
        application = ApplicationMock()
        # noinspection PyTypeChecker
        service = Service(application)

        import threading

        # Test service.start()
        thread = threading.Thread(target=service.start)
        thread.start()
        thread.join(timeout=0.1)

        self.assertIsInstance(service.server, ServerMock)

        # Test stop.stop()
        thread = threading.Thread(target=service.stop)
        thread.start()
        thread.join(timeout=0.1)

        # Test stop._on_shutdown()
        thread = threading.Thread(target=service._on_shutdown)
        thread.start()
        thread.join(timeout=0.1)

        self.assertIsNone(service.server)


class DefaultConfigTest(unittest.TestCase):
    def test_new_default_config(self):
        config = new_default_config(["/home/bibo/data/cube-1.zarr",
                                     "/home/bibo/data/cube-2.nc"],
                                    dict(conc_chl=(0.0, 20.0),
                                         conc_tsm=(0.0, 12.0, 'plasma')))
        self.assertEqual({
            'Datasets': [
                {
                    'FileSystem': 'file',
                    'Format': 'zarr',
                    'Identifier': 'dataset_1',
                    'Path': '/home/bibo/data/cube-1.zarr',
                    'Title': 'cube-1.zarr'
                },
                {
                    'FileSystem': 'file',
                    'Format': 'netcdf4',
                    'Identifier': 'dataset_2',
                    'Path': '/home/bibo/data/cube-2.nc',
                    'Title': 'cube-2.nc'
                }
            ],
            'Styles': [
                {'Identifier': 'default',
                 'ColorMappings': {
                     'conc_chl': {'ValueRange': [0.0, 20.0]},
                     'conc_tsm': {'ColorBar': 'plasma',
                                  'ValueRange': [0.0, 12.0]}},
                 }
            ]},
            config)

        with self.assertRaises(ValueError) as cm:
            new_default_config(["/home/bibo/data/cube-1.zarr",
                                "/home/bibo/data/cube-2.nc"],
                               dict(conc_chl=20.0,
                                    conc_tsm=(0.0, 12.0, 'plasma')))
        self.assertEqual("illegal style: conc_chl=20.0", f"{cm.exception}")


class UrlPatternTest(unittest.TestCase):
    def test_url_pattern_works(self):
        re_pattern = url_pattern('/open/{{id1}}ws/{{id2}}wf')
        matcher = re.fullmatch(re_pattern, '/open/34ws/a66wf')
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'id1': '34', 'id2': 'a66'})

        re_pattern = url_pattern('/open/ws{{id1}}/wf{{id2}}')
        matcher = re.fullmatch(re_pattern, '/open/ws34/wfa66')
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'id1': '34', 'id2': 'a66'})

        re_pattern = url_pattern('/datasets/{{ds_id}}/data.zarr/(?P<path>.*)')

        matcher = re.fullmatch(re_pattern, '/datasets/S2PLUS_2017/data.zarr/')
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'ds_id': 'S2PLUS_2017', 'path': ''})

        matcher = re.fullmatch(re_pattern, '/datasets/S2PLUS_2017/data.zarr/conc_chl/.zattrs')
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'ds_id': 'S2PLUS_2017', 'path': 'conc_chl/.zattrs'})

        x = 'C%3A%5CUsers%5CNorman%5CIdeaProjects%5Cccitools%5Cect-core%5Ctest%5Cui%5CTEST_WS_3'
        re_pattern = url_pattern('/ws/{{base_dir}}/res/{{res_name}}/add')
        matcher = re.fullmatch(re_pattern, '/ws/%s/res/SST/add' % x)
        self.assertIsNotNone(matcher)
        self.assertEqual(matcher.groupdict(), {'base_dir': x, 'res_name': 'SST'})

    def test_url_pattern_ok(self):
        self.assertEqual('/version',
                         url_pattern('/version'))
        self.assertEqual(r'(?P<num>[^\;\/\?\:\@\&\=\+\$\,]+)/get',
                         url_pattern('{{num}}/get'))
        self.assertEqual(r'/open/(?P<ws_name>[^\;\/\?\:\@\&\=\+\$\,]+)',
                         url_pattern('/open/{{ws_name}}'))
        self.assertEqual(r'/open/ws(?P<id1>[^\;\/\?\:\@\&\=\+\$\,]+)/wf(?P<id2>[^\;\/\?\:\@\&\=\+\$\,]+)',
                         url_pattern('/open/ws{{id1}}/wf{{id2}}'))
        self.assertEqual(r'/datasets/(?P<ds_id>[^\;\/\?\:\@\&\=\+\$\,]+)/data.zip/(.*)',
                         url_pattern('/datasets/{{ds_id}}/data.zip/(.*)'))

    def test_url_pattern_fail(self):
        with self.assertRaises(ValueError) as cm:
            url_pattern('/open/{{ws/name}}')
        self.assertEqual(str(cm.exception), 'name in {{name}} must be a valid identifier, but got "ws/name"')

        with self.assertRaises(ValueError) as cm:
            url_pattern('/info/{{id}')
        self.assertEqual(str(cm.exception), 'no matching "}}" after "{{" in "/info/{{id}"')


# test/webapi/ows/__init__.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.


# test/webapi/ows/wmts/__init__.py

# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.


# test/webapi/ows/wmts/test_routes.py
# The MIT License (MIT)
# Copyright (c) 2022 by the xcube team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

from ...helpers import ServerTest


class WmtsRoutesTest(ServerTest):

    def test_fetch_wmts_kvp_capabilities(self):
        response = self.fetch('/wmts/kvp'
                              '?SERVICE=WMTS'
                              '&VERSION=1.0.0'
                              '&REQUEST=GetCapabilities')
        self.assertResponseOK(response)

        response = self.fetch('/wmts/kvp'
                              '?service=WMTS'
                              '&version=1.0.0'
                              '&request=GetCapabilities')
        self.assertResponseOK(response)

        response = self.fetch('/wmts/kvp'
                              '?Service=WMTS'
                              '&Version=1.0.0'
                              '&Request=GetCapabilities')
        self.assertResponseOK(response)

        response = self.fetch('/wmts/kvp'
                              '?VERSION=1.0.0&REQUEST=GetCapabilities')
        self.assertBadRequestResponse(
            response,
            expected_message='value for "service" parameter must be "WMTS"'
        )

        response = self.fetch('/wmts/kvp'
                              '?SERVICE=WMS'
                              'VERSION=1.0.0'
                              '&REQUEST=GetCapabilities')
        self.assertBadRequestResponse(
            response,
            expected_message='value for "service" parameter must be "WMTS"'
        )

    def test_fetch_wmts_kvp_tile(self):
        response = self.fetch('/wmts/kvp'
                              '?Service=WMTS'
                              '&Version=1.0.0'
                              '&Request=GetTile'
                              '&Format=image/png'
                              '&Style=Default'
                              '&Layer=demo.conc_chl'
                              '&TileMatrixSet=WorldCRS84Quad'
                              '&TileMatrix=0'
                              '&TileRow=0'
                              '&TileCol=0')
        self.assertResponseOK(response)

        # issue #132 by Dirk
        response = self.fetch('/wmts/kvp'
                              '?Service=WMTS'
                              '&Version=1.0.0'
                              '&Request=GetTile'
                              '&Format=image/png'
                              '&Style=Default'
                              '&Layer=demo.conc_chl'
                              '&TileMatrixSet=WorldWebMercatorQuad'
                              '&TileMatrix=0'
                              '&TileRow=0'
                              '&TileCol=0'
                              '&Time=2017-01-25T09%3A35%3A50')
        self.assertResponseOK(response)

        # issue #132 by Dirk
        response = self.fetch('/wmts/kvp'
                              '?Service=WMTS'
                              '&Version=1.0.0'
                              '&Request=GetTile'
                              '&Format=image/png'
                              '&Style=Default'
                              '&Layer=demo.conc_chl'
                              '&TileMatrixSet=WorldWebMercatorQuad'
                              '&TileMatrix=0'
                              '&TileRow=0'
                              '&TileCol=0'
                              '&Time=2017-01-25T09%3A35%3A50%2F2017-01-25T10%3A20%3A15')
        self.assertResponseOK(response)

        response = self.fetch('/wmts/kvp'
                              '?Service=WMTS'
                              '&Version=1.0.0'
                              '&Request=GetTile'
                              '&Format=image/jpg'
                              '&Style=Default'
                              '&Layer=demo.conc_chl'
                              '&TileMatrixSet=WorldCRS84Quad'
                              '&TileMatrix=0'
                              '&TileRow=0'
                              '&TileCol=0')
        self.assertBadRequestResponse(
            response,
            'value for "format" parameter must be "image/png"'
        )

        response = self.fetch('/wmts/kvp'
                              '?Service=WMTS'
                              '&Version=1.1.0'
                              '&Request=GetTile'
                              '&Format=image/png'
                              '&Style=Default'
                              '&Layer=demo.conc_chl'
                              '&TileMatrixSet=WorldCRS84Quad'
                              '&TileMatrix=0'
                              '&TileRow=0'
                              '&TileCol=0')
        self.assertBadRequestResponse(
            response,
            'value for "version" parameter must be "1.0.0"'
        )

        response = self.fetch('/wmts/kvp'
                              '?Service=WMTS'
                              '&Request=GetTile'
                              '&Version=1.0.0'
                              '&Format=image/png'
                              '&Style=Default'
                              '&Layer=conc_chl'
                              '&TileMatrixSet=WorldCRS84Quad'
                              '&TileMatrix=0'
                              '&TileRow=0'
                              '&TileCol=0')
        self.assertBadRequestResponse(
            response,
            'value for "layer" parameter must be "<dataset>.<variable>"'
        )

        response = self.fetch('/wmts/kvp'
                              '?Service=WMTS'
                              '&Version=1.0.0'
                              '&Request=GetTile'
                              '&Format=image/png'
                              '&Style=Default'
                              '&Layer=demo.conc_chl'
                              '&TileMatrixSet=TileGrid_2000_1000'
                              '&TileMatrix=0'
                              '&TileRow=0'
                              '&TileCol=0')
        self.assertBadRequestResponse(
            response,
            'value for "tilematrixset" parameter must'
            ' be one of (\'WorldCRS84Quad\', \'WorldWebMercatorQuad\')'
        )

    def test_fetch_wmts_capabilities(self):
        response = self.fetch(
            '/wmts/1.0.0/WMTSCapabilities.xml')
        self.assertResponseOK(response)

    def test_fetch_wmts_tile(self):
        response = self.fetch(
            '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_geo(self):
        response = self.fetch(
            '/wmts/1.0.0/tile/demo/conc_chl/WorldCRS84Quad/0/0/0.png'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_mercator(self):
        response = self.fetch(
            '/wmts/1.0.0/tile/demo/conc_chl/WorldWebMercatorQuad/0/0/0.png'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_with_params(self):
        response = self.fetch(
            '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png'
            '?time=current&cbar=jet'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_with_params_geo(self):
        response = self.fetch(
            '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png'
            '?time=current&cbar=jet&TileMatrixSet=WorldCRS84Quad'
        )
        self.assertResponseOK(response)

    def test_fetch_wmts_tile_with_params_mercator(self):
        response = self.fetch(
            '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png'
            '?time=current&cbar=jet&TileMatrixSet=WorldWebMercatorQuad'
        )
        self.assertResponseOK(response)
