# docker-qgis/tests/test_qgis.py

import os
import shutil
import subprocess
import tempfile
import unittest


class QfcTestCase(unittest.TestCase):
    def test_package(self):
        project_directory = self.data_directory_path("simple_project")
        output_directory = tempfile.mkdtemp()

        command = [
            "docker-compose",
            "run",
            "--rm",
            "-v",
            "{}:/io/project/".format(project_directory),
            "-v",
            "{}:/io/output/".format(output_directory),
            "qgis",
            "bash",
            "-c",
            "./entrypoint.sh package /io/project/project.qgs /io/output",
        ]

        subprocess.check_call(
            command, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL
        )

        files = os.listdir(output_directory)

        self.assertIn("project_qfield.qgs", files)
        self.assertIn("france_parts_shape.shp", files)
        self.assertIn("france_parts_shape.dbf", files)
        self.assertIn("curved_polys.gpkg", files)
        self.assertIn("spatialite.db", files)

        shutil.rmtree(output_directory)

    def data_directory_path(self, path):
        basepath = os.path.dirname(os.path.abspath(__file__))
        return os.path.join(basepath, "testdata", path)


# docker-qgis/apply_deltas.py

#!/usr/bin/env python3

from enum import Enum
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union, cast

try:
    # 3.8
    from typing import TypedDict
except ImportError:
    # 3.7
    from typing_extensions import TypedDict

import argparse
import json
import logging
import shutil
import textwrap
import traceback
from functools import lru_cache
from pathlib import Path

import jsonschema

# pylint: disable=no-name-in-module
from qgis.core import (
    QgsDataSourceUri,
    QgsExpression,
    QgsFeature,
    QgsGeometry,
    QgsMapLayer,
    QgsMapLayerType,
    QgsProject,
    QgsProviderRegistry,
    QgsVectorLayer,
    QgsVectorLayerEditPassthrough,
    QgsVectorLayerUtils,
)
from qgis.PyQt.QtCore import QCoreApplication

logging.basicConfig(level=logging.DEBUG)

logger = logging.getLogger(__name__)
# /LOGGER

# TYPE DEFINITIONS
WKT = str
Uuid = str
FeaturePk = str
LayerId = str


class BaseOptions(TypedDict):
    cmd0: Callable
    project: str


class DeltaOptions(BaseOptions):
    delta_file: Optional[str]
    delta_contents: Optional[Dict]
    delta_log: Optional[str]
    overwrite_conflicts: bool
    inverse: bool
    transaction: bool


class DeltaMethod(str, Enum):
    def __str__(self):
        return str(self.value)

    CREATE = "create"
    PATCH = "patch"
    DELETE = "delete"


class DeltaExceptionType(str, Enum):
    def __str__(self):
        return str(self.value)

    Error = "ERROR"
    IO = "IO"
    Conflict = "CONFLICT"


class DeltaStatus(str, Enum):
    def __str__(self):
        return str(self.value)

    Applied = "status_applied"
    Conflict = "status_conflict"
    ApplyFailed = "status_apply_failed"
    UnknownError = "status_unknown_error"


class DeltaFeature(TypedDict):
    geometry: Optional[WKT]
    attributes: Optional[Dict[str, Any]]
    file_sha256: Optional[Dict[str, str]]


class Delta(TypedDict):
    id: Uuid
    localFk: FeaturePk
    sourceFk: FeaturePk
    localLayerId: LayerId
    sourceLayerId: LayerId
    method: DeltaMethod
    old: DeltaFeature
    new: DeltaFeature


class DeltaFile:
    def __init__(
        self,
        delta_file_id: str,
        project_id: str,
        version: str,
        deltas: List[Union[Dict, Delta]],
        files: List[str],
        client_pks: Dict[str, str],
    ):
        self.id = delta_file_id
        self.project_id = project_id
        self.version = version
        self.files = files
        self.client_pks = client_pks
        self.deltas: List[Delta] = []

        for d in deltas:
            self.deltas.append(cast(Delta, d))


# /TYPE DEFINITIONS


# EXCEPTION DEFINITIONS\
class DeltaException(Exception):
    def __init__(
        self,
        msg: str,
        e_type: DeltaExceptionType = DeltaExceptionType.Error,
        delta_file_id: str = None,
        layer_id: str = None,
        delta_idx: int = None,
        delta_id: str = None,
        feature_pk: str = None,
        modified_pk: str = None,
        conflicts: List[str] = None,
        method: DeltaMethod = None,
        provider_errors: str = None,
        descr: str = None,
    ):
        super(DeltaException, self).__init__(msg)
        self.e_type = e_type
        self.delta_file_id = delta_file_id
        self.layer_id = layer_id
        self.delta_idx = delta_idx
        self.delta_id = delta_id
        self.feature_pk = feature_pk
        self.modified_pk = modified_pk
        self.method = method
        self.conflicts = conflicts
        self.provider_errors = provider_errors
        self.descr = descr


# /EXCEPTION DEFINITIONS


BACKUP_SUFFIX = ".qfieldcloudbackup"
delta_log = []


def project_decorator(f):
    def wrapper(opts: BaseOptions, *args, **kw):
        project = QgsProject.instance()
        project.setAutoTransaction(opts["transaction"])
        project.read(opts.get("project_filename", opts["project"]))

        return f(project, opts, *args, **kw)  # type: ignore

    return wrapper


def delta_apply(
    project_filename: Path,
    delta_filename: Path,
    inverse: bool,
    overwrite_conflicts: bool,
):
    del delta_log[:]

    project = QgsProject.instance()
    logging.info(f'Loading project file "{project_filename}"...')
    project.read(str(project_filename))

    logging.info(f'Loading delta file "{delta_filename}"...')
    delta_file = delta_file_file_loader({"delta_file": delta_filename})  # type: ignore

    if not delta_file:
        raise Exception("Missing delta file")

    all_applied = apply_deltas_without_transaction(
        project, delta_file, inverse, overwrite_conflicts
    )

    project.clear()

    if not all_applied:
        logger.info("Some deltas have not been applied")

    delta_log_copy = [*delta_log]
    del delta_log[:]

    return delta_log_copy


@project_decorator
def cmd_delta_apply(project: QgsProject, opts: DeltaOptions) -> bool:
    accepted_state = None
    deltas = None
    has_uncaught_errors = False

    try:
        del delta_log[:]
        deltas = load_delta_file(opts)

        if opts["transaction"]:
            raise NotImplementedError(
                "Please check apply_deltas(project, deltas) and upgrade it, if needed"
            )
            # accepted_state = apply_deltas(project, deltas, inverse=opts['inverse'], overwrite_conflicts=opts['overwrite_conflicts'])
        else:
            accepted_state = apply_deltas_without_transaction(
                project,
                deltas,
                inverse=opts["inverse"],
                overwrite_conflicts=opts["overwrite_conflicts"],
            )

        project.clear()

    except Exception:
        exception_str = traceback.format_exc()
        logger.error("An unknown exception has occurred, check the deltalog carefully")
        logger.exception(exception_str)

        has_uncaught_errors = True

    deltas_count = len(deltas.deltas) if deltas is not None else "?"

    if accepted_state is None:
        logger.info(
            f"Uncaught exception occurred while trying to apply {deltas_count} deltas. "
            "Check the delta log if they have been processed and what is their status"
        )
    elif accepted_state:
        logger.info(f"All {deltas_count} deltas have been applied successfully")
    else:
        logger.info(
            f"Some of the {deltas_count} deltas have not been applied. "
            "Check the delta log if they have been processed and what is their status"
        )
    print("Delta log file contents:")
    print("========================")
    print(json.dumps(delta_log, indent=2, sort_keys=True, default=str))
    print("========================")

    if opts.get("delta_log"):
        with open(opts["delta_log"], "w") as f:
            json.dump(delta_log, f, indent=2, sort_keys=True, default=str)

    return has_uncaught_errors


@project_decorator
def cmd_backup_cleanup(project: QgsProject, opts: BaseOptions):
    for layer_id in project.mapLayers():
        layer = project.mapLayer(layer_id)

        assert layer

        if layer.type() != QgsMapLayerType.VectorLayer:
            continue

        backup_layer_path = get_backup_path(get_layer_path(layer))

        if backup_layer_path.exists():
            backup_layer_path.unlink()


@project_decorator
def cmd_backup_rollback(project: QgsProject, opts: BaseOptions):
    for layer_id in project.mapLayers():
        layer = project.mapLayer(layer_id)

        assert layer

        if layer.type() != QgsMapLayerType.VectorLayer:
            continue

        layer_path = get_layer_path(layer)
        backup_layer_path = get_backup_path(layer_path)

        assert layer_path.exists()

        if backup_layer_path.exists():
            backup_layer_path.unlink()


@lru_cache(maxsize=128)
def get_json_schema_validator() -> jsonschema.Draft7Validator:
    """Creates a JSON schema validator to check whether the provided delta
    file is valid. The function result is cached.

    Returns:
        jsonschema.Draft7Validator -- JSON Schema validator
    """
    with open("./schemas/deltafile_01.json") as f:
        schema_dict = json.load(f)

    jsonschema.Draft7Validator.check_schema(schema_dict)

    return jsonschema.Draft7Validator(schema_dict)


def delta_file_args_loader(args: DeltaOptions) -> Optional[DeltaFile]:
    """Get delta file contents as a dictionary passed in the args. Mostly used for testing.

    Arguments:
        args {DeltaOptions} -- main options

    Returns:
        Optional[DeltaFile] -- loaded delta file on success, otherwise none
    """
    if "delta_contents" not in args:
        return None

    obj = args["delta_contents"]
    get_json_schema_validator().validate(obj)
    delta_file = DeltaFile(
        obj["id"],
        obj["project"],
        obj["version"],
        obj["deltas"],
        obj["files"],
        obj["clientPks"],
    )

    return delta_file


def delta_file_file_loader(args: DeltaOptions) -> Optional[DeltaFile]:
    """Get delta file contents from a filesystem file.

    Arguments:
        args {DeltaOptions} -- main options

    Returns:
        Optional[DeltaFile] -- loaded delta file on success, otherwise none
    """
    if not isinstance(args.get("delta_file"), str):
        return None

    delta_file_path = Path(args["delta_file"])  # type: ignore
    delta_file: DeltaFile

    with delta_file_path.open("r") as f:
        obj = json.load(f)
        get_json_schema_validator().validate(obj)
        delta_file = DeltaFile(
            obj["id"],
            obj["project"],
            obj["version"],
            obj["deltas"],
            obj["files"],
            obj["clientPks"],
        )

    return delta_file


def load_delta_file(args: DeltaOptions) -> DeltaFile:
    """Loads delta file, using the provided {args}.

    Arguments:
        args {DeltaOptions} -- main options

    Returns:
        DeltaFile -- the loaded deltafile
    """
    deltas: Optional[DeltaFile] = None

    delta_file_loaders = [
        delta_file_args_loader,
        delta_file_file_loader,
    ]

    for delta_file_loader in delta_file_loaders:
        deltas = delta_file_loader(args)

        if deltas is not None:
            break

    assert deltas is not None, "Unable to load deltas"

    return deltas


def apply_deltas_without_transaction(
    project: QgsProject,
    delta_file: DeltaFile,
    inverse: bool = False,
    overwrite_conflicts: bool = False,
) -> bool:
    has_applied_all_deltas = True

    # apply deltas on each individual layer
    for idx, delta in enumerate(delta_file.deltas):
        delta_status = DeltaStatus.Applied
        layer_id: str = delta.get("sourceLayerId")
        layer: QgsVectorLayer = project.mapLayer(layer_id)
        feature = QgsFeature()

        try:
            if not isinstance(layer, QgsVectorLayer):
                raise DeltaException(f'No layer with id "{layer_id}"')

            if not layer.isValid():
                raise DeltaException(f'Invalid layer "{layer_id}"')

            if not layer.isEditable() and not layer.startEditing():
                raise DeltaException(
                    f'Cannot start editing layer "{layer_id}"',
                    provider_errors=layer.dataProvider().errors(),
                )

            has_edit_buffer = layer.editBuffer() and not isinstance(
                layer.editBuffer(), QgsVectorLayerEditPassthrough
            )
            delta = inverse_delta(delta) if inverse else delta

            if delta["method"] == str(DeltaMethod.CREATE):
                # don't use the returned feature as the PK might contain the "Autogenerated" string value, instead the real one
                created_feature = create_feature(
                    layer, delta, overwrite_conflicts=overwrite_conflicts
                )

                # apparently the only way to obtain the feature if there is no edit buffer is use the returned created_feature
                if not has_edit_buffer:
                    feature = created_feature
            elif delta["method"] == str(DeltaMethod.PATCH):
                feature = patch_feature(
                    layer,
                    delta,
                    overwrite_conflicts=overwrite_conflicts,
                    client_pks=delta_file.client_pks,
                )
            elif delta["method"] == str(DeltaMethod.DELETE):
                feature = delete_feature(
                    layer,
                    delta,
                    overwrite_conflicts=overwrite_conflicts,
                    client_pks=delta_file.client_pks,
                )
            else:
                raise DeltaException("Unknown delta method")

            def committed_features_added_cb(layer_id, features):
                if len(features) != 0 and len(features) != 1:
                    raise DeltaException(
                        f"Expected only one feature, but actually {len(features)} were added."
                    )

                if layer_id != layer.id():
                    raise DeltaException(
                        f"Expected the layer with the added layer to be {layer.id()}, but got {layer_id}."
                    )

                nonlocal feature
                feature = features[0]

            if has_edit_buffer:
                # in QGIS the only way to get the real features that have been added after commit, if edit buffer is present, is to use this signal.
                layer.committedFeaturesAdded.connect(committed_features_added_cb)

            if not layer.commitChanges():
                raise DeltaException(
                    "Failed to commit changes",
                    provider_errors=layer.dataProvider().errors(),
                )

            if has_edit_buffer:
                QCoreApplication.processEvents()
                layer.committedFeaturesAdded.disconnect(committed_features_added_cb)

            logger.info(f'Successfully applied delta on layer "{layer_id}"')

            feature_pk = delta.get("sourcePk")
            modified_pk = None
            if feature.isValid():
                _pk_attr_idx, pk_attr_name = find_layer_pk(layer)

                if not pk_attr_name:
                    raise DeltaException(f'Layer "{layer.name()}" has no primary key.')

                modified_pk = feature.attribute(pk_attr_name)

                if modified_pk and modified_pk != str(feature_pk):
                    logger.warning(
                        f'The modified feature pk valued does not match "sourcePk" in the delta in "{layer_id}": sourcePk={feature_pk} modifiedFeaturePk={modified_pk}'
                    )
            else:
                logger.warning(
                    f'The returned modified feature is invalid in "{layer_id}"'
                )

            delta_log.append(
                {
                    "msg": "Successfully applied delta!",
                    "status": delta_status,
                    "e_type": None,
                    "delta_file_id": delta_file.id,
                    "layer_id": layer_id,
                    "delta_index": idx,
                    "delta_id": delta["uuid"],
                    "feature_pk": feature_pk,
                    "modified_pk": modified_pk,
                    "conflicts": None,
                    "provider_errors": None,
                    "method": delta["method"],
                }
            )

        except DeltaException as err:
            err.layer_id = err.layer_id or layer_id
            err.delta_file_id = err.delta_file_id or delta_file.id
            err.delta_idx = err.delta_idx or idx
            err.delta_id = err.delta_id or delta["uuid"]
            err.feature_pk = err.feature_pk or delta.get("sourcePk")
            err.method = err.method or delta.get("method")

            if err.e_type == DeltaExceptionType.Conflict:
                delta_status = DeltaStatus.Conflict
                logger.warning(f"Conflicts while applying a single delta: {err}")
            else:
                delta_status = DeltaStatus.ApplyFailed
                logger.warning(f"Error while applying a single delta: {err}")

            if layer is not None and not layer.rollBack():
                logger.error(f'Failed to rollback layer "{layer_id}": {err}')

            has_applied_all_deltas = False
            delta_log.append(
                {
                    "msg": str(err),
                    "status": delta_status,
                    "e_type": err.e_type,
                    "delta_file_id": err.delta_file_id,
                    "layer_id": err.layer_id,
                    "delta_index": err.delta_idx,
                    "delta_id": err.delta_id,
                    "feature_pk": err.feature_pk,
                    "modified_pk": err.modified_pk,
                    "conflicts": err.conflicts,
                    "provider_errors": err.provider_errors,
                    "method": err.method,
                }
            )
        except Exception as err:
            delta_status = DeltaStatus.UnknownError
            delta_log.append(
                {
                    "msg": str(err),
                    "status": delta_status,
                    "e_type": None,
                    "delta_file_id": delta_file.id,
                    "layer_id": layer_id,
                    "delta_index": idx,
                    "delta_id": delta.get("uuid"),
                    "feature_pk": None,
                    "modified_pk": None,
                    "conflicts": None,
                    "provider_errors": None,
                    "method": delta.get("method"),
                }
            )

            logger.error(
                f"An unknown error has been encountered while applying delta: {err}"
            )

            raise err

    return has_applied_all_deltas


def apply_deltas(
    project: QgsProject,
    delta_file: DeltaFile,
    inverse: bool = False,
    overwrite_conflicts: bool = False,
) -> DeltaStatus:
    """Applies the deltas from a loaded delta file on the layers in a project.

    The general algorithm is as follows:
    1) group all individual deltas by layer id.
    2) make a backup of the layer data source. In case things go wrong, one
    can rollback from them.
    3) apply deltas on each individual layer:
    startEditing -> apply changes -> commit
    4) if all is good, delete the backup files.

    Arguments:
        project {QgsProject} -- project
        delta_file {DeltaFile} -- delta file
        inverse {bool} -- inverses the direction of the deltas. Makes the
        delta `old` to `new` and `new` to `old`. Mainly used to rollback the
        applied changes using the same delta file.
        overwrite_conflicts {bool} -- whether the conflicts are ignored

    Returns:
        bool -- indicates whether a conflict occurred
    """
    has_conflict = False
    deltas_by_layer: Dict[LayerId, List[Delta]] = {}
    layers_by_id: Dict[LayerId, QgsVectorLayer] = {}
    transcation_by_layer: Dict[str, str] = {}
    opened_transactions: Dict[str, List[LayerId]] = {}

    for layer_id in project.mapLayers():
        layer = project.mapLayer(layer_id)
        conn_type = layer.providerType()
        conn_string = QgsDataSourceUri(layer.source()).connectionInfo(False)

        if len(conn_string) == 0:
            continue

        # here we use '$$$$$' as a separator, nothing special, can be easily changed to any other string
        transaction_id = conn_type + "$$$$$" + conn_string
        transcation_by_layer[layer_id] = transaction_id
        opened_transactions[transaction_id] = opened_transactions.get(
            transaction_id, []
        )
        opened_transactions[transaction_id].append(layer_id)

    # group all individual deltas by layer id
    for d in delta_file.deltas:
        layer_id = d["sourceLayerId"]
        deltas_by_layer[layer_id] = deltas_by_layer.get(layer_id, [])
        deltas_by_layer[layer_id].append(d)
        layers_by_id[layer_id] = project.mapLayer(layer_id)

        if not isinstance(layers_by_id[layer_id], QgsVectorLayer):
            raise DeltaException(
                "The layer does not exist: {}".format(layer_id), layer_id=layer_id
            )

    # make a backup of the layer data source.
    for layer_id in layers_by_id.keys():
        if not is_layer_file_based(layers_by_id[layer_id]):
            continue

        layer_path = get_layer_path(layers_by_id[layer_id])
        layer_backup_path = get_backup_path(layer_path)

        assert layer_path.exists()
        # TODO enable this when needed
        # assert not layer_backup_path.exists()

        if not shutil.copyfile(layer_path, layer_backup_path):
            raise DeltaException(
                "Unable to backup file for layer {}".format(layer_id),
                layer_id=layer_id,
                e_type=DeltaExceptionType.IO,
            )

    modified_layer_ids: Set[LayerId] = set()

    # apply deltas on each individual layer
    for layer_id in deltas_by_layer.keys():
        # keep the try/except block inside the loop, so we can have the `layer_id` context
        try:
            if apply_deltas_on_layer(
                layers_by_id[layer_id],
                deltas_by_layer[layer_id],
                inverse,
                overwrite_conflicts=overwrite_conflicts,
            ):
                has_conflict = True

            modified_layer_ids.add(layer_id)
        except DeltaException as err:
            rollback_deltas(layers_by_id)
            err.layer_id = err.layer_id or layer_id
            err.delta_file_id = err.delta_file_id or delta_file.id
            raise err
        except Exception as err:
            rollback_deltas(layers_by_id)
            raise DeltaException("Failed to apply changes") from err

    committed_layer_ids: Set[LayerId] = set()

    for layer_id in deltas_by_layer.keys():
        # keep the try/except block inside the loop, so we can have the `layer_id` context
        try:
            # the layer has already been commited. This might happend if there are multiple layers in the same transaction group.
            if layer_id in committed_layer_ids:
                continue

            if layers_by_id[layer_id].commitChanges():
                transaction_id = transcation_by_layer.get(layer_id)

                if transaction_id and transaction_id in opened_transactions:
                    committed_layer_ids = set(
                        [*committed_layer_ids, *opened_transactions[transaction_id]]
                    )
                    del opened_transactions[transaction_id]
                else:
                    committed_layer_ids.add(layer_id)
            else:
                raise DeltaException("Failed to commit")
        except DeltaException:
            # all the modified layers must be rollbacked
            for layer_id in modified_layer_ids - committed_layer_ids:
                if not layers_by_id[layer_id].rollBack():
                    logger.warning("Failed to rollback")

            rollback_deltas(layers_by_id, committed_layer_ids=committed_layer_ids)

    if not cleanup_backups(set(layers_by_id.keys())):
        logger.warning("Failed to cleanup backups, other than that - success")

    return DeltaStatus.Conflict if has_conflict else DeltaStatus.Applied


def rollback_deltas(
    layers_by_id: Dict[LayerId, QgsVectorLayer],
    committed_layer_ids: Set[LayerId] = set(),
) -> bool:
    """Rollback applied deltas by restoring the layer data source backup files.

    Arguments:
        layers_by_id {Dict[LayerId, QgsVectorLayer]} -- layers
        committed_layer_ids {Set[LayerId]} -- layer ids to be rollbacked.

    Returns:
        bool -- whether there were no rollback errors. If True, the state of
        the project might be broken, but the old data is preserved in the
        backup files.
    """
    is_success = True
    # we need to keep the backups of `committed_layer_ids` in case something goes wrong
    backups_to_remove_layer_ids = set(layers_by_id.keys()) - committed_layer_ids

    # first rollback the buffer of each layer
    for layer in layers_by_id.values():
        if layer.isEditable():
            if layer.rollBack():
                logger.warning("Unable to rollback layer {}".format(layer.id()))

    # if there are already committed layers, restore the backup
    for layer_id in committed_layer_ids:
        if not layers_by_id.get(layer_id):
            logger.warning(
                "Unable to restore original layer file, path missing: {}".format(
                    layer_id
                )
            )
            continue

        layer_path = get_layer_path(layers_by_id[layer_id])
        layer_backup_path = get_backup_path(layer_path)

        # NOTE pathlib operations will raise in case of error
        try:
            layer_backup_path.rename(layer_path)
        except Exception as err:
            # TODO nothing better to do here?
            is_success = False
            logger.warning(
                "Unable to restore original layer file: {}. Reason: {}".format(
                    layer_id, err
                )
            )

    # no mater what, try to cleanup the backups that are no longer needed.
    # this way it would be easier to restore the original state.
    cleanup_backups(backups_to_remove_layer_ids)

    return is_success


def cleanup_backups(layer_paths: Set[str]) -> bool:
    """Cleanup the layer backups. Attempts to remove all backup files, whether
    or not there is an error.

    Arguments:
        layer_paths {Set[str]} -- layer paths, which should have their
        backups removed

    Returns:
        bool -- whether all paths are successfully removed.
    """
    is_success = True

    for layer_path in layer_paths:
        layer_path = Path(layer_path)
        layer_backup_path = get_backup_path(layer_path)
        try:
            if layer_backup_path.exists():
                layer_backup_path.unlink()
        except Exception as err:
            is_success = False
            logger.warning(
                "Unable to remove backup: {}. Reason: {}".format(layer_backup_path, err)
            )

    return is_success


def apply_deltas_on_layer(
    layer: QgsVectorLayer,
    deltas: List[Delta],
    inverse: bool,
    should_commit: bool = False,
    overwrite_conflicts: bool = False,
) -> bool:
    """Applies the deltas on the layer provided.

    Arguments:
        layer {QgsVectorLayer} -- target layer
        deltas {List[Delta]} -- ordered list of deltas to be applied
        inverse {bool} -- inverses the direction of the deltas. Makes the
        delta `old` to `new` and `new` to `old`. Mainly used to rollback the
        applied changes using the same delta file.

    Keyword Arguments:
        should_commit {bool} -- whether the changes should be committed
        (default: {False})
        overwrite_conflicts {bool} -- whether the conflicts are ignored

    Raises:
        DeltaException: whenever the changes cannot be applied

    Returns:
        bool -- indicates whether a conflict occurred
    """
    assert layer is not None
    assert layer.type() == QgsMapLayerType.VectorLayer

    has_conflict = False

    if not layer.isEditable() and not layer.startEditing():
        raise DeltaException("Cannot start editing")

    for idx, d in enumerate(deltas):
        assert d["sourceLayerId"] == layer.id()

        delta = inverse_delta(d) if inverse else d

        try:
            if delta["method"] == str(DeltaMethod.CREATE):
                create_feature(layer, delta, overwrite_conflicts=overwrite_conflicts)
            elif delta["method"] == str(DeltaMethod.PATCH):
                patch_feature(layer, delta, overwrite_conflicts=overwrite_conflicts)
            elif delta["method"] == str(DeltaMethod.DELETE):
                delete_feature(layer, delta, overwrite_conflicts=overwrite_conflicts)
            else:
                raise DeltaException("Unknown delta method")
        except DeltaException as err:
            # TODO I am lazy now and all these properties are set only here, but should be moved in depth
            err.layer_id = err.layer_id or layer.id()
            err.delta_idx = err.delta_idx or idx
            err.delta_id = err.delta_id or delta.get("id")
            err.method = err.method or delta.get("method")
            err.feature_pk = err.feature_pk or delta.get("sourcePk")
            err.provider_errors = err.provider_errors or layer.dataProvider().errors()

            if err.e_type == DeltaExceptionType.Conflict:
                has_conflict = True
                logger.warning(
                    "Conflicts while applying a single delta: {}".format(str(err)), err
                )
            else:
                if not layer.rollBack():
                    # So unfortunate situation, that the only safe thing to do is to cancel the whole script
                    raise DeltaException(
                        "Cannot rollback layer changes: {}".format(layer.id())
                    ) from err

                raise err
        except Exception as err:
            if not layer.rollBack():
                # So unfortunate situation, that the only safe thing to do is to cancel the whole script
                raise DeltaException(
                    "Cannot rollback layer changes: {}".format(layer.id())
                ) from err

            raise DeltaException(
                "An error has been encountered while applying delta:"
                + str(err).replace("\n", ""),
                layer_id=layer.id(),
                delta_idx=idx,
                delta_id=delta.get("id"),
                method=delta.get("method"),
                descr=traceback.format_exc(),
                feature_pk=delta.get("sourcePk"),
            ) from err

    if should_commit and not layer.commitChanges():
        if not layer.rollBack():
            # So unfortunate situation, that the only safe thing to do is to cancel the whole script
            raise DeltaException("Cannot rollback layer changes: {}".format(layer.id()))

        raise DeltaException("Cannot commit changes")

    return has_conflict


def find_layer_pk(layer: QgsVectorLayer) -> Tuple[int, str]:
    fields = layer.fields()
    pk_attrs = [*layer.primaryKeyAttributes(), fields.indexFromName("fid")]
    # we assume the first index to be the primary key index... kinda stupid, but memory layers don't have primary key at all, but we use it on geopackages, but... snap!
    pk_attr_idx = pk_attrs[0]

    if pk_attr_idx == -1:
        return (-1, "")

    pk_attr_name = fields.at(pk_attr_idx).name()

    return (pk_attr_idx, pk_attr_name)


def get_feature(
    layer: QgsVectorLayer, delta: Delta, client_pks: Dict[str, str] = None
) -> QgsFeature:
    _pk_attr_idx, pk_attr_name = find_layer_pk(layer)

    assert pk_attr_name

    source_pk = delta["sourcePk"]

    if client_pks:
        client_pk_key = f'{delta["clientId"]}__{delta["localPk"]}'
        if client_pk_key in client_pks:
            source_pk = client_pks[client_pk_key]

    expr = " {} = {} ".format(
        QgsExpression.quotedColumnRef(pk_attr_name),
        QgsExpression.quotedValue(source_pk),
    )

    feature = QgsFeature()
    has_feature = False
    for f in layer.getFeatures(expr):
        if has_feature:
            raise Exception("More than one feature match the feature select query")

        feature = f
        has_feature = True

    return feature


def create_feature(
    layer: QgsVectorLayer, delta: Delta, overwrite_conflicts: bool
) -> QgsFeature:
    """Creates new feature in layer

    Arguments:
        layer {QgsVectorLayer} -- target layer. Must be in editing mode!
        delta {Delta} -- delta describing the created feature
        overwrite_conflicts {bool} -- if there are conflicts with an existing feature, ignore them

    Raises:
        DeltaException: whenever the feature cannot be created
    """
    fields = layer.fields()
    new_feat_delta = delta["new"]
    geometry = QgsGeometry()

    if "geometry" in new_feat_delta:
        if isinstance(new_feat_delta["geometry"], str):
            geometry = QgsGeometry.fromWkt(new_feat_delta["geometry"])
        elif new_feat_delta["geometry"] is not None:
            logger.warning("The provided geometry is not null or a WKT string.")

    new_feat_attrs = new_feat_delta.get("attributes", {})
    feat_attrs = {}

    if new_feat_attrs:
        # `fid` is an extra field created during conversion to gpkg and makes this assert to fail.
        # if fields.size() < len(new_feat_attrs):
        #     raise DeltaException('The layer has less attributes than the provided by the delta')

        if new_feat_attrs:
            for field in fields:
                attr_name = field.name()

                if attr_name in new_feat_attrs:
                    attr_value = new_feat_attrs[attr_name]
                    attr_index = fields.indexFromName(attr_name)
                    feat_attrs[attr_index] = attr_value

    new_feat = QgsVectorLayerUtils.createFeature(layer, geometry, feat_attrs)

    if not new_feat.isValid():
        raise DeltaException("Unable to create a valid feature")

    if not layer.addFeature(new_feat):
        raise DeltaException(
            "Unable to add new feature", provider_errors=layer.dataProvider().errors()
        )

    return new_feat


def patch_feature(
    layer: QgsVectorLayer,
    delta: Delta,
    overwrite_conflicts: bool,
    client_pks: Dict[str, str],
) -> QgsFeature:
    """Patches a feature in layer

    Arguments:
        layer {QgsVectorLayer} -- target layer. Must be in edit mode!
        delta {Delta} -- delta describing the patch
        overwrite_conflicts {bool} -- if there are conflicts with an existing feature, ignore them

    Raises:
        DeltaException: whenever the feature cannot be patched
    """
    new_feature_delta = delta["new"]
    old_feature_delta = delta["old"]
    old_feature = get_feature(layer, delta, client_pks)

    if not old_feature.isValid():
        raise DeltaException("Unable to find feature")

    conflicts = compare_feature(old_feature, old_feature_delta, True)

    if len(conflicts) != 0:
        if overwrite_conflicts:
            logger.warning(
                f'Conflicts while applying delta "{delta["uuid"]}". Ignoring since `overwrite_conflicts` flag set to `True`.\nConflicts:\n{conflicts}'
            )
        else:
            raise DeltaException(
                "There are conflicts with the already existing feature!",
                conflicts=conflicts,
                e_type=DeltaExceptionType.Conflict,
            )

    geometry = None

    if "geometry" in new_feature_delta:
        if layer.isSpatial():
            if isinstance(new_feature_delta["geometry"], str):
                geometry = QgsGeometry.fromWkt(new_feature_delta["geometry"])

                if geometry.isNull() or geometry.type() != layer.geometryType():
                    raise DeltaException(
                        "The provided geometry type differs from the layer geometry type"
                    )
            elif new_feature_delta["geometry"] is None:
                geometry = QgsGeometry()
            else:
                logger.warning(
                    "The provided geometry is not null or a WKT string, ignoring geometry."
                )

            if new_feature_delta["geometry"] == old_feature_delta.get("geometry"):
                logger.warning(
                    "The geometries of the new and the old features are the same, even though by spec they should not be provided in such case. Ignoring geometry."
                )

            if geometry is not None:
                if not layer.changeGeometry(old_feature.id(), geometry, True):
                    raise DeltaException(
                        "Unable to change geometry",
                        provider_errors=layer.dataProvider().errors(),
                    )
        else:
            logger.warning("Layer is not spatial, ignoring geometry")

    fields = layer.fields()
    new_attrs = new_feature_delta.get("attributes") or {}
    old_attrs = old_feature_delta.get("attributes") or {}

    for attr_name, new_attr_value in new_attrs.items():
        # NOTE the old_attrs may be missing or empty
        old_attr_value = old_attrs.get(attr_name)

        if new_attrs[attr_name] == old_attr_value:
            logger.warning(
                "The delta has features with the same value in both old and new values"
            )
            continue

        if not layer.changeAttributeValue(
            old_feature.id(),
            fields.indexOf(attr_name),
            new_attr_value,
            old_attrs[attr_name],
            True,
        ):
            raise DeltaException(
                'Unable to change attribute "{}"'.format(attr_name),
                provider_errors=layer.dataProvider().errors(),
            )

    return layer.getFeature(old_feature.id())


def delete_feature(
    layer: QgsVectorLayer,
    delta: Delta,
    overwrite_conflicts: bool,
    client_pks: Dict[str, str],
) -> QgsFeature:
    """Deletes a feature from layer

    Arguments:
        layer {QgsVectorLayer} -- target layer. Must be in edit mode!
        delta {Delta} -- delta describing the deleted feature
        overwrite_conflicts {bool} -- if there are conflicts with an existing feature, ignore them

    Raises:
        DeltaException: whenever the feature cannot be deleted
    """
    old_feature_delta = delta["old"]
    old_feature = get_feature(layer, delta, client_pks)

    if not old_feature.isValid():
        raise DeltaException("Unable to find feature")

    conflicts = compare_feature(old_feature, old_feature_delta)

    if len(conflicts) != 0:
        if overwrite_conflicts:
            logger.warning(
                f'Conflicts while applying delta "{delta["uuid"]}". Ignoring since `overwrite_conflicts` flag set to `True`.\nConflicts:\n{conflicts}'
            )
        else:
            raise DeltaException(
                "There are conflicts with the already existing feature!",
                conflicts=conflicts,
                e_type=DeltaExceptionType.Conflict,
            )

    if not layer.deleteFeature(old_feature.id()):
        raise DeltaException("Unable delete feature")

    return old_feature


def compare_feature(
    feature: QgsFeature, delta_feature: DeltaFeature, is_delta_subset: bool = False
) -> List[str]:
    """Compares a feature with delta description of a feature and reports the
    differences. Checks both the geometry and the attributes. If
    {is_delta_subset} is true, allows the delta attributes to be subset of the
    {feature} attributes.

    Arguments:
        feature {QgsFeature} -- target feature
        delta_feature {DeltaFeature} -- target delta description of a feature

    Keyword Arguments:
        is_delta_subset {bool} -- whether to precisely match the delta attributes
        on the original feature (default: {False})

    Returns:
        List[str] -- a list of differences found
    """
    # NOTE this can be something more structured, JSON like object.
    # However, I think we should not record the diff/conflicts, as they may change if the underlying feature is updated.
    conflicts: List[str] = []

    # TODO enable once we are done
    # if delta_feature.get('geometry') != feature.geometry().asWkt(17):
    #     conflicts.append('Geometry missmatch')

    delta_feature_attrs: Optional[Dict[str, Any]] = delta_feature.get("attributes")

    if delta_feature_attrs:
        delta_feature_attr_names = delta_feature_attrs.keys()
        feature_attr_names = feature.fields().names()

        # TODO reenable this, when it is clear what we do when there is property mismatch
        # if not is_delta_subset:
        #     for attr in feature_attr_names:
        #         if attr not in delta_feature_attr_names:
        #             conflicts.append(f'The attribute "{attr}" in the original feature is not available in the delta')

        for attr in delta_feature_attr_names:
            if attr not in feature_attr_names:
                # TODO reenable this, when it is clear what we do when there is property mismatch
                # conflicts.append(f'The attribute "{attr}" in the delta is not available in the original feature')
                continue

            if feature.attribute(attr) != delta_feature_attrs[attr]:
                conflicts.append(
                    f'The attribute "{attr}" that has a conflict:\n-{delta_feature_attrs[attr]}\n+{feature.attribute(attr)}'
                )

    return conflicts


@lru_cache(maxsize=128)
def get_layer_path(layer: QgsMapLayer) -> Path:
    """Returns a `Path` object of the data source filename of the given layer.

    Arguments:
        layer {QgsMapLayer} -- target layer

    Returns:
        Path -- `Path` object
    """
    data_source_uri = layer.dataProvider().dataSourceUri()
    data_source_decoded = QgsProviderRegistry.instance().decodeUri(
        "ogr", data_source_uri
    )
    return Path(data_source_decoded["path"])


def get_backup_path(path: Path) -> Path:
    """Returns a `Path` object of with backup suffix

    Arguments:
        path {Path} -- target path

    Returns:
        Path -- suffixed path
    """
    return Path(str(path) + BACKUP_SUFFIX)


def is_layer_file_based(layer: QgsMapLayer) -> bool:
    return len(str(get_layer_path(layer))) == 0


def inverse_delta(delta: Delta) -> Delta:
    """Returns shallow copy of the delta with reversed `old` and `new` keys

    Arguments:
        delta {Delta} -- delta

    Returns:
        [type] -- reversed delta
    """
    copy: Dict[str, Any] = {**delta}
    copy["old"], copy["new"] = delta.get("new"), delta.get("old")

    if copy["method"] == DeltaMethod.CREATE.name:
        copy["method"] = DeltaMethod.DELETE.name
    elif copy["method"] == DeltaMethod.DELETE.name:
        copy["method"] = DeltaMethod.CREATE.name

    return cast(Delta, copy)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog="COMMAND",
        description="",
        formatter_class=argparse.RawDescriptionHelpFormatter,  # type: ignore
        epilog=textwrap.dedent(
            """
            example:
                # apply deltas on a project
                ./apply_delta.py delta apply ./path/to/project.qgs ./path/to/delta.json

                # rollback deltas on a project
                ./apply_delta.py delta apply --inverse ./path/to/project.qgs ./path/to/delta.json

                # rollback deltas on a project
                ./apply_delta.py backup cleanup
        """
        ),
    )
    parser.add_argument(
        "--skip-csv-header",
        action="store_true",
        help="Skip writing CSV header in the log file",
    )

    subparsers = parser.add_subparsers(dest="cmd0")

    # deltas
    parser_delta = subparsers.add_parser(
        "delta", help="rollback a delta file on a project"
    )
    delta_subparsers = parser_delta.add_subparsers(dest="cmd1")

    parser_delta_apply = delta_subparsers.add_parser(
        "apply", help="apply a delta file on a project"
    )
    parser_delta_apply.add_argument("project", type=str, help="Path to QGIS project")
    parser_delta_apply.add_argument("delta_file", type=str, help="Path to delta file")
    parser_delta_apply.add_argument(
        "--delta-log", type=str, help="Path to delta log file"
    )
    parser_delta_apply.add_argument(
        "--overwrite-conflicts",
        action="store_true",
        help="Apply deltas even if there are conflicts.",
    )
    parser_delta_apply.add_argument(
        "--inverse",
        action="store_true",
        help="Inverses the direction of the deltas. Makes the delta `old` to `new` and `new` to `old`. Mainly used to rollback the applied changes using the same delta file..",
    )
    parser_delta_apply.add_argument(
        "--transaction",
        action="store_true",
        help='Apply individual deltas in the deltafile in the "all-or-nothing" manner, with transaction mode enabled',
    )
    parser_delta_apply.set_defaults(func=cmd_delta_apply)
    # /deltas

    # backup
    parser_backup = subparsers.add_parser(
        "backup", help="rollback a delta file on a project"
    )
    backup_subparsers = parser_backup.add_subparsers(dest="cmd1")

    parser_backup_cleanup = backup_subparsers.add_parser(
        "cleanup", help="rollback a delta file on a project"
    )
    parser_backup_cleanup.add_argument("project", type=str, help="Path to QGIS project")
    parser_backup_cleanup.add_argument(
        "--transaction",
        action="store_true",
        help='Apply individual deltas in the deltafile in the "all-or-nothing" manner, with transaction mode enabled',
    )
    parser_backup_cleanup.set_defaults(func=cmd_backup_cleanup)

    parser_backup_rollback = backup_subparsers.add_parser(
        "rollback", help="rollback a delta file on a project"
    )
    parser_backup_rollback.add_argument(
        "project", type=str, help="Path to QGIS project"
    )
    parser_backup_rollback.add_argument(
        "--transaction",
        action="store_true",
        help='Apply individual deltas in the deltafile in the "all-or-nothing" manner, with transaction mode enabled',
    )
    parser_backup_rollback.set_defaults(func=cmd_backup_rollback)
    # /backup

    args = parser.parse_args()

    if "func" in args:
        args.func(vars(args))  # type: ignore
    else:
        parser.parse_args(["-h"])


# docker-qgis/entrypoint.py

#!/usr/bin/env python3

import argparse
import logging
import os
from pathlib import Path
from typing import Dict, Union

import qfieldcloud.qgis.apply_deltas
import qfieldcloud.qgis.process_projectfile
from libqfieldsync.offline_converter import ExportType, OfflineConverter
from libqfieldsync.project import ProjectConfiguration
from libqfieldsync.utils.file_utils import get_project_in_folder
from qfieldcloud.qgis.utils import (
    Step,
    StepOutput,
    WorkDirPath,
    Workflow,
    get_layers_data,
    layers_data_to_string,
)
from qgis.core import (
    QgsCoordinateTransform,
    QgsOfflineEditing,
    QgsProject,
    QgsRectangle,
    QgsVectorLayer,
)

PGSERVICE_FILE_CONTENTS = os.environ.get("PGSERVICE_FILE_CONTENTS")

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


def _call_qfieldsync_packager(project_filename: Path, package_dir: Path) -> str:
    """Call the function of QFieldSync to package a project for QField"""

    project = QgsProject.instance()
    if not project_filename.exists():
        raise FileNotFoundError(project_filename)

    if not project.read(str(project_filename)):
        raise Exception(f"Unable to open file with QGIS: {project_filename}")

    layers = project.mapLayers()
    project_config = ProjectConfiguration(project)
    vl_extent_wkt = QgsRectangle()
    vl_extent_crs = project.crs().authid()

    if project_config.area_of_interest and project_config.area_of_interest_crs:
        vl_extent_wkt = project_config.area_of_interest
        vl_extent_crs = project_config.area_of_interest_crs
    else:
        vl_extent = QgsRectangle()
        for layer in layers.values():
            if type(layer) != QgsVectorLayer:
                continue

            layer_extent = layer.extent()

            if layer_extent.isNull() or not layer_extent.isFinite():
                continue

            try:
                transform = QgsCoordinateTransform(layer.crs(), project.crs(), project)
                vl_extent.combineExtentWith(
                    transform.transformBoundingBox(layer_extent)
                )
            except Exception as err:
                logger.error(
                    "Failed to transform the bbox for layer {} from {} to {} CRS.".format(
                        layer.name(), layer.crs(), project.crs()
                    ),
                    exc_info=err,
                )

        if vl_extent.isNull() or not vl_extent.isFinite():
            logger.info("Failed to obtain the project extent from project layers.")

            try:
                vl_extent = qfieldcloud.qgis.utils.extract_project_details(project)[
                    "extent"
                ]
                vl_extent = QgsRectangle.fromWkt(vl_extent)
            except Exception as err:
                logger.error(
                    "Failed to get the project extent from the current map canvas.",
                    exc_info=err,
                )

        if vl_extent.isNull() or not vl_extent.isFinite():
            raise Exception("Failed to obtain the project extent.")

        # sometimes the result is a polygon whose all points are on the same line
        # this is an invalid polygon and cannot libqfieldsync does not like it
        vl_extent = vl_extent.buffered(1)
        vl_extent_wkt = vl_extent.asWktPolygon()
        vl_extent_crs = project.crs().authid()

    offline_editing = QgsOfflineEditing()
    offline_converter = OfflineConverter(
        project,
        str(package_dir),
        vl_extent_wkt,
        vl_extent_crs,
        offline_editing,
        export_type=ExportType.Cloud,
        create_basemap=False,
    )

    # Disable the basemap generation because it needs the processing
    # plugin to be installed
    offline_converter.project_configuration.create_base_map = False
    offline_converter.convert()

    packaged_project_filename = get_project_in_folder(str(package_dir))
    if Path(packaged_project_filename).stat().st_size == 0:
        raise Exception("The packaged QGIS project file is empty.")

    return packaged_project_filename


def _extract_layer_data(project_filename: Union[str, Path]) -> Dict:
    project_filename = str(project_filename)
    project = QgsProject.instance()
    project.read(project_filename)
    layers_by_id = get_layers_data(project)

    logging.info(
        f"QGIS project layer checks\n{layers_data_to_string(layers_by_id)}",
    )

    return layers_by_id


def cmd_package_project(args):
    workflow = Workflow(
        id="package_project",
        name="Package Project",
        version="2.0",
        description="Packages a QGIS project to be used on QField. Converts layers for offline editing if configured.",
        steps=[
            Step(
                id="start_qgis_app",
                name="Start QGIS Application",
                method=qfieldcloud.qgis.utils.start_app,
            ),
            Step(
                id="download_project_directory",
                name="Download Project Directory",
                arguments={
                    "project_id": args.projectid,
                    "destination": WorkDirPath(mkdir=True),
                    "skip_attachments": True,
                },
                method=qfieldcloud.qgis.utils.download_project,
                return_names=["tmp_project_dir"],
            ),
            Step(
                id="qgis_layers_data",
                name="QGIS Layers Data",
                arguments={
                    "project_filename": WorkDirPath("files", args.project_file),
                },
                method=_extract_layer_data,
                return_names=["layers_by_id"],
                outputs=["layers_by_id"],
            ),
            Step(
                id="package_project",
                name="Package Project",
                arguments={
                    "project_filename": WorkDirPath("files", args.project_file),
                    "package_dir": WorkDirPath("export", mkdir=True),
                },
                method=_call_qfieldsync_packager,
                return_names=["qfield_project_filename"],
            ),
            Step(
                id="qfield_layer_data",
                name="Packaged Layers Data",
                arguments={
                    "project_filename": StepOutput(
                        "package_project", "qfield_project_filename"
                    ),
                },
                method=_extract_layer_data,
                return_names=["layers_by_id"],
                outputs=["layers_by_id"],
            ),
            Step(
                id="stop_qgis_app",
                name="Stop QGIS Application",
                method=qfieldcloud.qgis.utils.stop_app,
            ),
            Step(
                id="upload_packaged_project",
                name="Upload Packaged Project",
                arguments={
                    "project_id": args.projectid,
                    "package_dir": WorkDirPath("export", mkdir=True),
                },
                method=qfieldcloud.qgis.utils.upload_package,
            ),
        ],
    )

    qfieldcloud.qgis.utils.run_workflow(
        workflow,
        Path("/io/feedback.json"),
    )


def cmd_apply_deltas(args):
    workflow = Workflow(
        id="apply_changes",
        name="Apply Changes",
        version="2.0",
        steps=[
            Step(
                id="start_qgis_app",
                name="Start QGIS Application",
                method=qfieldcloud.qgis.utils.start_app,
            ),
            Step(
                id="download_project_directory",
                name="Download Project Directory",
                arguments={
                    "project_id": args.projectid,
                    "destination": WorkDirPath(mkdir=True),
                    "skip_attachments": True,
                },
                method=qfieldcloud.qgis.utils.download_project,
                return_names=["tmp_project_dir"],
            ),
            Step(
                id="apply_deltas",
                name="Apply Deltas",
                arguments={
                    "project_filename": WorkDirPath("files", args.project_file),
                    "delta_filename": "/io/deltafile.json",
                    "inverse": args.inverse,
                    "overwrite_conflicts": args.overwrite_conflicts,
                },
                method=qfieldcloud.qgis.apply_deltas.delta_apply,
                return_names=["delta_feedback"],
                outputs=["delta_feedback"],
            ),
            Step(
                id="stop_qgis_app",
                name="Stop QGIS Application",
                method=qfieldcloud.qgis.utils.stop_app,
            ),
            Step(
                id="upload_exported_project",
                name="Upload Project",
                arguments={
                    "project_id": args.projectid,
                    "project_dir": WorkDirPath("files"),
                },
                method=qfieldcloud.qgis.utils.upload_project,
            ),
        ],
    )

    qfieldcloud.qgis.utils.run_workflow(
        workflow,
        Path("/io/feedback.json"),
    )


def cmd_process_projectfile(args):
    workflow = Workflow(
        id="process_projectfile",
        name="Process Projectfile",
        version="2.0",
        steps=[
            Step(
                id="start_qgis_app",
                name="Start QGIS Application",
                method=qfieldcloud.qgis.utils.start_app,
            ),
            Step(
                id="download_project_directory",
                name="Download Project Directory",
                arguments={
                    "project_id": args.projectid,
                    "destination": WorkDirPath(mkdir=True),
                    "skip_attachments": True,
                },
                method=qfieldcloud.qgis.utils.download_project,
                return_names=["tmp_project_dir"],
            ),
            Step(
                id="project_validity_check",
                name="Project Validity Check",
                arguments={
                    "project_filename": WorkDirPath("files", args.project_file),
                },
                method=qfieldcloud.qgis.process_projectfile.check_valid_project_file,
            ),
            Step(
                id="opening_check",
                name="Opening Check",
                arguments={
                    "project_filename": WorkDirPath("files", args.project_file),
                },
                method=qfieldcloud.qgis.process_projectfile.load_project_file,
                return_names=["project"],
            ),
            Step(
                id="project_details",
                name="Project Details",
                arguments={
                    "project": StepOutput("opening_check", "project"),
                },
                method=qfieldcloud.qgis.process_projectfile.extract_project_details,
                return_names=["project_details"],
                outputs=["project_details"],
            ),
            Step(
                id="generate_thumbnail_image",
                name="Generate Thumbnail Image",
                arguments={
                    "project": StepOutput("opening_check", "project"),
                    "thumbnail_filename": Path("/io/thumbnail.png"),
                },
                method=qfieldcloud.qgis.process_projectfile.generate_thumbnail,
            ),
            Step(
                id="stop_qgis_app",
                name="Stop QGIS Application",
                method=qfieldcloud.qgis.utils.stop_app,
            ),
        ],
    )

    qfieldcloud.qgis.utils.run_workflow(
        workflow,
        Path("/io/feedback.json"),
    )


if __name__ == "__main__":

    # Set S3 logging levels
    logging.getLogger("nose").setLevel(logging.CRITICAL)
    logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
    logging.getLogger("urllib3").setLevel(logging.CRITICAL)

    if PGSERVICE_FILE_CONTENTS:
        with open(Path.home().joinpath(".pg_service.conf"), "w") as f:
            f.write(PGSERVICE_FILE_CONTENTS)

    parser = argparse.ArgumentParser(prog="COMMAND")

    subparsers = parser.add_subparsers(dest="cmd")

    parser_package = subparsers.add_parser("package", help="Package a project")
    parser_package.add_argument("projectid", type=str, help="projectid")
    parser_package.add_argument("project_file", type=str, help="QGIS project file path")
    parser_package.set_defaults(func=cmd_package_project)

    parser_delta = subparsers.add_parser("delta_apply", help="Apply deltafile")
    parser_delta.add_argument("projectid", type=str, help="projectid")
    parser_delta.add_argument("project_file", type=str, help="QGIS project file path")
    parser_delta.add_argument(
        "--overwrite-conflicts", dest="overwrite_conflicts", action="store_true"
    )
    parser_delta.add_argument("--inverse", dest="inverse", action="store_true")
    parser_delta.set_defaults(func=cmd_apply_deltas)

    parser_process_projectfile = subparsers.add_parser(
        "process_projectfile", help="Process QGIS project file"
    )
    parser_process_projectfile.add_argument("projectid", type=str, help="projectid")
    parser_process_projectfile.add_argument(
        "project_file", type=str, help="QGIS project file path"
    )
    parser_process_projectfile.set_defaults(func=cmd_process_projectfile)

    args = parser.parse_args()
    args.func(args)


# docker-qgis/process_projectfile.py

import logging
import sys
from pathlib import Path
from typing import Dict
from xml.etree import ElementTree

from qfieldcloud.qgis.utils import BaseException, get_layers_data, layers_data_to_string
from qgis.core import QgsMapRendererParallelJob, QgsMapSettings, QgsProject
from qgis.PyQt.QtCore import QEventLoop, QSize
from qgis.PyQt.QtGui import QColor

logging.basicConfig(
    stream=sys.stderr, level=logging.DEBUG, format="%(asctime)s %(levelname)s %(msg)s"
)


class ProjectFileNotFoundException(BaseException):
    message = 'Project file "%(project_filename)s" does not exist'


class InvalidFileExtensionException(BaseException):
    message = (
        'Project file "%(project_filename)s" has unknown file extension "%(extension)s"'
    )


class InvalidXmlFileException(BaseException):
    message = "Project file is an invalid XML document:\n%(xml_error)s"


class InvalidQgisFileException(BaseException):
    message = 'Project file "%(project_filename)s" is invalid QGIS file:\n%(error)s'


class InvalidLayersException(BaseException):
    message = 'Project file "%(project_filename)s" contains invalid layers'


class FailedThumbnailGenerationException(BaseException):
    message = "Failed to generate project thumbnail:\n%(reason)s"


def check_valid_project_file(project_filename: Path) -> None:
    logging.info("Check QGIS project file validity...")

    if not project_filename.exists():
        raise ProjectFileNotFoundException(project_filename=project_filename)

    if project_filename.suffix == ".qgs":
        try:
            with open(project_filename) as f:
                ElementTree.fromstring(f.read())
        except ElementTree.ParseError as err:
            raise InvalidXmlFileException(
                project_filename=project_filename, xml_error=err
            )
    elif project_filename.suffix != ".qgz":
        raise InvalidFileExtensionException(
            project_filename=project_filename, extension=project_filename.suffix
        )


def load_project_file(project_filename: Path) -> QgsProject:
    logging.info("Open QGIS project file...")

    project = QgsProject.instance()
    if not project.read(str(project_filename)):
        raise InvalidXmlFileException(error=project.error())

    return project


def extract_project_details(project: QgsProject) -> Dict[str, str]:
    """Extract project details"""
    logging.info("Extract project details...")

    details = {}

    logging.info("Reading QGIS project file...")
    map_settings = QgsMapSettings()

    def on_project_read(doc):
        r, _success = project.readNumEntry("Gui", "/CanvasColorRedPart", 255)
        g, _success = project.readNumEntry("Gui", "/CanvasColorGreenPart", 255)
        b, _success = project.readNumEntry("Gui", "/CanvasColorBluePart", 255)
        background_color = QColor(r, g, b)
        map_settings.setBackgroundColor(background_color)

        details["background_color"] = background_color.name()

        nodes = doc.elementsByTagName("mapcanvas")

        for i in range(nodes.size()):
            node = nodes.item(i)
            element = node.toElement()
            if (
                element.hasAttribute("name")
                and element.attribute("name") == "theMapCanvas"
            ):
                map_settings.readXml(node)

        map_settings.setRotation(0)
        map_settings.setOutputSize(QSize(1024, 768))

        details["extent"] = map_settings.extent().asWktPolygon()

    project.readProject.connect(on_project_read)
    project.read(project.fileName())

    details["crs"] = project.crs().authid()
    details["project_name"] = project.title()

    logging.info("Extracting layer and datasource details...")

    details["layers_by_id"] = get_layers_data(project)
    details["ordered_layer_ids"] = list(details["layers_by_id"].keys())

    logging.info(
        f'QGIS project layer checks\n{layers_data_to_string(details["layers_by_id"])}',
    )

    return details


def generate_thumbnail(project: QgsProject, thumbnail_filename: Path) -> None:
    """Create a thumbnail for the project

    As from https://docs.qgis.org/3.16/en/docs/pyqgis_developer_cookbook/composer.html#simple-rendering

    Args:
        project (QgsProject)
        thumbnail_filename (Path)
    """
    logging.info("Generate project thumbnail image...")

    map_settings = QgsMapSettings()
    layer_tree = project.layerTreeRoot()

    def on_project_read(doc):
        r, _success = project.readNumEntry("Gui", "/CanvasColorRedPart", 255)
        g, _success = project.readNumEntry("Gui", "/CanvasColorGreenPart", 255)
        b, _success = project.readNumEntry("Gui", "/CanvasColorBluePart", 255)
        map_settings.setBackgroundColor(QColor(r, g, b))

        nodes = doc.elementsByTagName("mapcanvas")

        for i in range(nodes.size()):
            node = nodes.item(i)
            element = node.toElement()
            if (
                element.hasAttribute("name")
                and element.attribute("name") == "theMapCanvas"
            ):
                map_settings.readXml(node)

        map_settings.setRotation(0)
        map_settings.setTransformContext(project.transformContext())
        map_settings.setPathResolver(project.pathResolver())
        map_settings.setOutputSize(QSize(100, 100))
        map_settings.setLayers(reversed(list(layer_tree.customLayerOrder())))
        # print(f'output size: {map_settings.outputSize().width()} {map_settings.outputSize().height()}')
        # print(f'layers: {[layer.name() for layer in map_settings.layers()]}')

    project.readProject.connect(on_project_read)
    project.read(project.fileName())

    renderer = QgsMapRendererParallelJob(map_settings)

    event_loop = QEventLoop()
    renderer.finished.connect(event_loop.quit)
    renderer.start()

    event_loop.exec_()

    img = renderer.renderedImage()

    if not img.save(str(thumbnail_filename)):
        raise FailedThumbnailGenerationException(reason="Failed to save.")


# docker-qgis/utils.py

import atexit
import inspect
import json
import logging
import os
import socket
import subprocess
import sys
import tempfile
import traceback
import uuid
from contextlib import contextmanager
from datetime import datetime
from pathlib import Path
from typing import IO, Any, Callable, Dict, List, Optional, Union

from libqfieldsync.layer import LayerSource
from qfieldcloud_sdk import sdk
from qgis.core import (
    Qgis,
    QgsApplication,
    QgsMapLayer,
    QgsMapSettings,
    QgsProject,
    QgsProviderRegistry,
)
from qgis.PyQt import QtCore, QtGui
from tabulate import tabulate

# Get environment variables
JOB_ID = os.environ.get("JOB_ID")

qgs_stderr_logger = logging.getLogger("QGIS_STDERR")
qgs_stderr_logger.setLevel(logging.DEBUG)
qgs_msglog_logger = logging.getLogger("QGIS_MSGLOG")
qgs_msglog_logger.setLevel(logging.DEBUG)


def _qt_message_handler(mode, context, message):
    log_level = logging.DEBUG
    if mode == QtCore.QtDebugMsg:
        log_level = logging.DEBUG
    elif mode == QtCore.QtInfoMsg:
        log_level = logging.INFO
    elif mode == QtCore.QtWarningMsg:
        log_level = logging.WARNING
    elif mode == QtCore.QtCriticalMsg:
        log_level = logging.CRITICAL
    elif mode == QtCore.QtFatalMsg:
        log_level = logging.FATAL

    qgs_stderr_logger.log(
        log_level,
        message,
        extra={
            "time": datetime.now().isoformat(),
            "line": context.line,
            "file": context.file,
            "function": context.function,
        },
    )


QtCore.qInstallMessageHandler(_qt_message_handler)


def _write_log_message(message, tag, level):
    log_level = logging.DEBUG

    # in 3.16 it was Qgis.None, but since None is a reserved keyword, it was inaccessible
    try:
        Qgis.NoLevel
    except Exception:
        Qgis.NoLevel = 4

    if level == Qgis.NoLevel:
        log_level = logging.DEBUG
    elif level == Qgis.Info:
        log_level = logging.INFO
    elif level == Qgis.Success:
        log_level = logging.INFO
    elif level == Qgis.Warning:
        log_level = logging.WARNING
    elif level == Qgis.Critical:
        log_level = logging.CRITICAL

    qgs_msglog_logger.log(
        log_level,
        message,
        extra={
            "time": datetime.now().isoformat(),
            "tag": tag,
        },
    )


QGISAPP: QgsApplication = None


def start_app():
    """
    Will start a QgsApplication and call all initialization code like
    registering the providers and other infrastructure. It will not load
    any plugins.

    You can always get the reference to a running app by calling `QgsApplication.instance()`.

    The initialization will only happen once, so it is safe to call this method repeatedly.

        Returns
        -------
        QgsApplication

        A QgsApplication singleton
    """
    global QGISAPP

    if QGISAPP is None:
        qgs_stderr_logger.info(
            f"Starting QGIS app version {Qgis.versionInt()} ({Qgis.devVersion()})..."
        )
        argvb = []

        # Note: QGIS_PREFIX_PATH is evaluated in QgsApplication -
        # no need to mess with it here.
        gui_flag = False
        QGISAPP = QgsApplication(argvb, gui_flag)

        QtCore.qInstallMessageHandler(_qt_message_handler)
        os.environ["QGIS_CUSTOM_CONFIG_PATH"] = tempfile.mkdtemp("", "QGIS_CONFIG")
        QGISAPP.initQgis()

        QtCore.qInstallMessageHandler(_qt_message_handler)
        QgsApplication.messageLog().messageReceived.connect(_write_log_message)

        # make sure the app is closed, otherwise the container exists with non-zero
        @atexit.register
        def exitQgis():
            stop_app()

    return QGISAPP


def stop_app():
    """
    Cleans up and exits QGIS
    """
    global QGISAPP

    # note that if this function is called from @atexit.register, the globals are cleaned up
    if "QGISAPP" not in globals():
        return

    QgsProject.instance().read("")

    if QGISAPP is not None:
        qgs_stderr_logger.info("Stopping QGIS app…")
        QGISAPP.exitQgis()
        del QGISAPP


def download_project(
    project_id: str, destination: Path = None, skip_attachments: bool = True
) -> Path:
    """Download the files in the project "working" directory from the S3
    Storage into a temporary directory. Returns the directory path"""
    if not destination:
        # Create a temporary directory
        destination = Path(tempfile.mkdtemp())

    # Create a local working directory
    working_dir = destination.joinpath("files")
    working_dir.mkdir(parents=True)

    client = sdk.Client()
    files = client.list_remote_files(project_id)

    if skip_attachments:
        files = [file for file in files if not file["is_attachment"]]

    client.download_files(
        files,
        project_id,
        sdk.FileTransferType.PROJECT,
        str(working_dir),
        filter_glob="*",
        throw_on_error=True,
        show_progress=False,
    )

    return destination


def upload_package(project_id: str, package_dir: Path) -> None:
    client = sdk.Client()
    client.upload_files(
        project_id,
        sdk.FileTransferType.PACKAGE,
        str(package_dir),
        filter_glob="*",
        throw_on_error=True,
        show_progress=False,
        job_id=JOB_ID,
    )


def upload_project(project_id: str, project_dir: Path) -> None:
    """Upload the files from the `project_dir` to the permanent file storage."""
    client = sdk.Client()
    client.upload_files(
        project_id,
        sdk.FileTransferType.PROJECT,
        str(project_dir),
        filter_glob="*",
        throw_on_error=True,
        show_progress=False,
    )


class WorkflowValidationException(Exception):
    ...


class Workflow:
    def __init__(
        self,
        id: str,
        version: str,
        name: str,
        steps: List["Step"],
        description: str = "",
    ):
        self.id = id
        self.version = version
        self.name = name
        self.description = description
        self.steps = steps

        self.validate()

    def validate(self):
        if not self.steps:
            raise WorkflowValidationException(
                f'The workflow "{self.id}" should contain at least one step.'
            )

        all_step_returns = {}
        for step in self.steps:
            param_names = []
            sig = inspect.signature(step.method)
            for param in sig.parameters.values():
                if (
                    param.kind != inspect.Parameter.KEYWORD_ONLY
                    and param.kind != inspect.Parameter.POSITIONAL_OR_KEYWORD
                ):
                    raise WorkflowValidationException(
                        f'The workflow "{self.id}" method "{step.method.__name__}" has a non keyword parameter "{param.name}".'
                    )

                if param.name not in step.arguments:
                    raise WorkflowValidationException(
                        f'The workflow "{self.id}" method "{step.method.__name__}" has an argument "{param.name}" that is not available in the step definition "arguments", expected one of {list(step.arguments.keys())}.'
                    )

                param_names.append(param.name)

            for name, value in step.arguments.items():
                if isinstance(value, StepOutput):
                    if value.step_id not in all_step_returns:
                        raise WorkflowValidationException(
                            f'The workflow "{self.id}" has step "{step.id}" that requires a non-existing step return value "{value.step_id}.{value.return_name}" for argument "{name}". Previous step with that id does not exist.'
                        )

                    if value.return_name not in all_step_returns[value.step_id]:
                        raise WorkflowValidationException(
                            f'The workflow "{self.id}" has step "{step.id}" that requires a non-existing step return value "{value.step_id}.{value.return_name}" for argument "{name}". Previous step with that id found, but returns no value with such name.'
                        )

                if name not in param_names:
                    raise WorkflowValidationException(
                        f'The workflow "{self.id}" method "{step.method.__name__}" receives a parameter "{name}" that is not available in the method definition, expected one of {param_names}.'
                    )

            all_step_returns[step.id] = all_step_returns.get(step.id, step.return_names)


class Step:
    def __init__(
        self,
        id: str,
        name: str,
        method: Callable,
        arguments: Dict[str, Any] = {},
        return_names: List[str] = [],
        outputs: List[str] = [],
    ):
        self.id = id
        self.name = name
        self.method = method
        self.arguments = arguments
        # names of method return values
        self.return_names = return_names
        # names of method return values that will be part of the outputs. They are assumed to be safe to be shown to the user.
        self.outputs = outputs
        self.stage = 0


class StepOutput:
    def __init__(self, step_id: str, return_name: str):
        self.step_id = step_id
        self.return_name = return_name


class WorkDirPath:
    def __init__(self, *parts: str, mkdir: bool = False) -> None:
        self.parts = parts
        self.mkdir = mkdir

    def eval(self, root: Path) -> Path:
        path = root.joinpath(*self.parts)

        if self.mkdir:
            path.mkdir(parents=True, exist_ok=True)

        return path


class BaseException(Exception):
    """QFieldCloud Exception"""

    message = ""

    def __init__(self, message: str = None, **kwargs):
        self.message = (message or self.message) % kwargs
        self.details = kwargs

        super().__init__(self.message)


@contextmanager
def logger_context(step: Step):
    log_uuid = uuid.uuid4()

    try:
        # NOTE we are still using the reference from the `steps` list
        step.stage = 1
        print(f"::<<<::{log_uuid} {step.name}", file=sys.stderr)
        yield
        step.stage = 2
    finally:
        print(f"::>>>::{log_uuid} {step.stage}", file=sys.stderr)


def is_localhost(hostname: str, port: int = None) -> bool:
    """returns True if the hostname points to the localhost, otherwise False."""
    if port is None:
        port = 22  # no port specified, lets just use the ssh port
    try:
        hostname = socket.getfqdn(hostname)
        if hostname in ("localhost", "0.0.0.0"):
            return True
        localhost = socket.gethostname()
        localaddrs = socket.getaddrinfo(localhost, port)
        targetaddrs = socket.getaddrinfo(hostname, port)
        for (_family, _socktype, _proto, _canonname, sockaddr) in localaddrs:
            for (_rfamily, _rsocktype, _rproto, _rcanonname, rsockaddr) in targetaddrs:
                if rsockaddr[0] == sockaddr[0]:
                    return True
        return False
    except Exception:
        return False


def has_ping(hostname: str) -> bool:
    ping = subprocess.Popen(
        ["ping", "-c", "1", "-w", "5", hostname],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )

    out, error = ping.communicate()

    return not bool(error) and "100% packet loss" not in out.decode("utf8")


def get_layer_filename(layer: QgsMapLayer) -> Optional[str]:
    metadata = QgsProviderRegistry.instance().providerMetadata(
        layer.dataProvider().name()
    )

    if metadata is not None:
        decoded = metadata.decodeUri(layer.source())
        if "path" in decoded:
            return decoded["path"]

    return None


def extract_project_details(project: QgsProject) -> Dict[str, str]:
    """Extract project details"""
    map_settings = QgsMapSettings()
    details = {}

    def on_project_read(doc):
        r, _success = project.readNumEntry("Gui", "/CanvasColorRedPart", 255)
        g, _success = project.readNumEntry("Gui", "/CanvasColorGreenPart", 255)
        b, _success = project.readNumEntry("Gui", "/CanvasColorBluePart", 255)
        background_color = QtGui.QColor(r, g, b)
        map_settings.setBackgroundColor(background_color)

        details["background_color"] = background_color.name()

        nodes = doc.elementsByTagName("mapcanvas")

        for i in range(nodes.size()):
            node = nodes.item(i)
            element = node.toElement()
            if (
                element.hasAttribute("name")
                and element.attribute("name") == "theMapCanvas"
            ):
                map_settings.readXml(node)

        map_settings.setRotation(0)
        map_settings.setOutputSize(QtCore.QSize(1024, 768))

        details["extent"] = map_settings.extent().asWktPolygon()

    project.readProject.connect(on_project_read)
    project.read(project.fileName())

    details["crs"] = project.crs().authid()
    details["project_name"] = project.title()

    return details


def json_default(obj):
    obj_str = type(obj).__qualname__
    try:
        obj_str += f" {str(obj)}"
    except Exception:
        obj_str += " <non-representable>"
    return f"<non-serializable: {obj_str}>"


def run_workflow(
    workflow: Workflow,
    feedback_filename: Optional[Union[IO, Path]],
) -> Dict:
    """Executes the steps required to run a task and return structured feedback from the execution

    Each step has a method that is executed.
    Method may take arguments as defined in `arguments` and ordered in `arg_names`.
    Method may return values, as defined in `return_values`.
    Some return values can used as task output, as defined in `output_names`.
    Some return values can used as arguments for next steps, as defined in `public_returns`.

    Args:
        workflow (Workflow): workflow to be executed
        feedback_filename (Optional[Union[IO, Path]]): write feedback to an IO device, to Path filename, or don't write it
    """
    feedback: Dict[str, Any] = {
        "feedback_version": "2.0",
        "workflow_version": workflow.version,
        "workflow_id": workflow.id,
        "workflow_name": workflow.name,
    }
    # it may be modified after the successful completion of each step.
    step_returns = {}

    try:
        root_workdir = Path(tempfile.mkdtemp())
        for step in workflow.steps:
            with logger_context(step):
                arguments = {
                    **step.arguments,
                }
                for name, value in arguments.items():
                    if isinstance(value, StepOutput):
                        arguments[name] = step_returns[value.step_id][value.return_name]
                    elif isinstance(value, WorkDirPath):
                        arguments[name] = value.eval(root_workdir)

                return_values = step.method(**arguments)
                return_values = (
                    return_values if len(step.return_names) > 1 else (return_values,)
                )

                step_returns[step.id] = {}
                for name, value in zip(step.return_names, return_values):
                    step_returns[step.id][name] = value

    except Exception as err:
        feedback["error"] = str(err)
        (_type, _value, tb) = sys.exc_info()
        feedback["error_stack"] = traceback.format_tb(tb)
    finally:
        feedback["steps"] = []
        feedback["outputs"] = {}

        for step in workflow.steps:
            step_feedback = {
                "id": step.id,
                "name": step.name,
                "stage": step.stage,
                "returns": {},
            }

            if step.stage == 2:
                step_feedback["returns"] = step_returns[step.id]
                feedback["outputs"][step.id] = {}
                for output_name in step.outputs:
                    feedback["outputs"][step.id][output_name] = step_returns[step.id][
                        output_name
                    ]

            feedback["steps"].append(step_feedback)

        if feedback_filename in [sys.stderr, sys.stdout]:
            print("Feedback:")
            print(
                json.dump(
                    feedback,
                    feedback_filename,
                    indent=2,
                    sort_keys=True,
                    default=json_default,
                )
            )
        elif isinstance(feedback_filename, Path):
            with open(feedback_filename, "w") as f:
                json.dump(feedback, f, indent=2, sort_keys=True, default=json_default)

        return feedback


def get_layers_data(project: QgsProject) -> Dict[str, Dict]:
    layers_by_id = {}

    for layer in project.mapLayers().values():
        error = layer.error()
        layer_id = layer.id()
        layer_source = LayerSource(layer)
        layers_by_id[layer_id] = {
            "id": layer_id,
            "name": layer.name(),
            "crs": layer.crs().authid() if layer.crs() else None,
            "wkb_type": layer.wkbType()
            if layer.type() == QgsMapLayer.VectorLayer
            else None,
            "qfs_action": layer.customProperty("QFieldSync/action"),
            "qfs_cloud_action": layer.customProperty("QFieldSync/cloud_action"),
            "qfs_is_geometry_locked": layer.customProperty(
                "QFieldSync/is_geometry_locked"
            ),
            "qfs_photo_naming": layer.customProperty("QFieldSync/photo_naming"),
            "is_valid": layer.isValid(),
            "datasource": layer.dataProvider().uri().uri()
            if layer.dataProvider()
            else None,
            "type": layer.type(),
            "type_name": layer.type().name,
            "error_code": "no_error",
            "error_summary": error.summary() if error.messageList() else "",
            "error_message": layer.error().message(),
            "filename": layer_source.filename,
            "provider_error_summary": None,
            "provider_error_message": None,
        }

        if layers_by_id[layer_id]["is_valid"]:
            continue

        data_provider = layer.dataProvider()

        if data_provider:
            data_provider_error = data_provider.error()

            if data_provider.isValid():
                # there might be another reason why the layer is not valid, other than the data provider
                layers_by_id[layer_id]["error_code"] = "invalid_layer"
            else:
                layers_by_id[layer_id]["error_code"] = "invalid_dataprovider"

            layers_by_id[layer_id]["provider_error_summary"] = (
                data_provider_error.summary()
                if data_provider_error.messageList()
                else ""
            )
            layers_by_id[layer_id][
                "provider_error_message"
            ] = data_provider_error.message()

            if not layers_by_id[layer_id]["provider_error_summary"]:
                service = data_provider.uri().service()
                if service:
                    layers_by_id[layer_id][
                        "provider_error_summary"
                    ] = f'Unable to connect to service "{service}".'

                host = data_provider.uri().host()
                port = (
                    int(data_provider.uri().port())
                    if data_provider.uri().port()
                    else None
                )
                if host and (is_localhost(host, port) or has_ping(host)):
                    layers_by_id[layer_id][
                        "provider_error_summary"
                    ] = f'Unable to connect to host "{host}".'

                path = layer_source.metadata.get("path")
                if path and not os.path.exists(path):
                    layers_by_id[layer_id]["error_summary"] = f'File "{path}" missing.'

        else:
            layers_by_id[layer_id]["error_code"] = "missing_dataprovider"
            layers_by_id[layer_id][
                "provider_error_summary"
            ] = "No data provider available"

    return layers_by_id


def layers_data_to_string(layers_by_id):
    # Print layer check results
    table = [
        [
            d["name"],
            f'...{d["id"][-6:]}',
            d["is_valid"],
            d["error_code"],
            d["error_summary"],
            d["provider_error_summary"],
        ]
        for d in layers_by_id.values()
    ]

    return tabulate(
        table,
        headers=[
            "Layer Name",
            "Layer Id",
            "Is Valid",
            "Status",
            "Error Summary",
            "Provider Summary",
        ],
    )


# scripts/check_envvars.py

#!/bin/env python3
import argparse
import re
from pathlib import Path
from typing import Dict, List, Set

import yaml


def get_env_varnames_from_envfile(filename: str) -> Set[str]:
    result = set()

    with open(filename, "r") as f:
        for line in f.readlines():
            if line.strip().startswith("#"):
                continue

            if len(line.strip()) == 0:
                continue

            variable_name = line.strip().split("=")[0]

            # not settings
            if variable_name in ["COMPOSE_FILE", "COMPOSE_PATH_SEPARATOR"]:
                continue

            result.add(variable_name)

        return result


def get_env_varnames_from_docker_compose(filename: Path) -> Set[str]:
    regex = r"\$\{(\w+)\}"
    result = set()

    with open(filename, "r") as f:
        for line in f.readlines():
            if line.strip().startswith("#"):
                continue

            for match in re.finditer(regex, line):
                result.add(match.group(1))

    return result


def get_env_varnames_from_docker_compose_files(
    search_path: str,
) -> Dict[str, List[str]]:
    env_vars = {}

    for path in Path(search_path).glob("**/docker-compose*.yml"):
        assert path.is_file()

        for varname in get_env_varnames_from_docker_compose(path):
            occurrences = env_vars.get(varname, [])
            occurrences.append(path.name)
            env_vars[varname] = occurrences

    return env_vars


def get_env_varnames_from_k8s_kustomization(filename: Path) -> Set[str]:
    result = set()

    with open(filename, "r") as f:
        k8s_config = yaml.load(f, Loader=yaml.SafeLoader)
        for config in k8s_config["configMapGenerator"]:
            for envvar_and_val in config["literals"]:
                result.add(envvar_and_val.strip().split("=")[0])

    return result


def get_env_varnames_from_k8s_secrets(filename: Path) -> Set[str]:
    with open(filename, "r") as f:
        config = yaml.load(f, Loader=yaml.SafeLoader)
        return set(config["spec"]["encryptedData"].keys())


def get_env_varnames_from_k8s_environments(search_path: str) -> Dict[str, List[str]]:
    env_vars = {}

    for path in Path(search_path).iterdir():
        if not path.is_dir():
            continue

        kustomization_varnames = get_env_varnames_from_k8s_kustomization(
            path.joinpath("kustomization.yml")
        )
        secret_varnames = get_env_varnames_from_k8s_secrets(path.joinpath("secret.yml"))

        common_varnames = kustomization_varnames.intersection(secret_varnames)
        assert (
            len(common_varnames) == 0
        ), f"Envvar(s) {common_varnames} are present both in the kustomization and the secret file in {path.name} environment. Choose only one!"

        for varname in kustomization_varnames.union(secret_varnames):
            occurrences = env_vars.get(varname, [])
            occurrences.append(path.name)
            env_vars[varname] = occurrences

    return env_vars


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog="check_envvars",
        epilog="./check_envvars.py .env.example --docker-compose-dir . --k8s-environments-dir ./deployment/kubernetes/environments",
    )
    parser.add_argument("envfile", type=str, help="Environment file.")
    parser.add_argument(
        "--docker-compose-dir",
        type=str,
        help="Directory containing docker-compose*.yml files.",
    )
    parser.add_argument(
        "--k8s-environments-dir",
        type=str,
        help="Directory containing k8s configuration and secret files.",
    )
    parser.add_argument(
        "--ignored-varnames",
        type=str,
        nargs="*",
        help="Ignored varnames.",
    )
    args = parser.parse_args()

    problems = []
    envfile_vars = get_env_varnames_from_envfile(args.envfile)
    ignored_varnames = args.ignored_varnames or []

    if args.docker_compose_dir:
        dockercompose_vars = get_env_varnames_from_docker_compose_files(
            args.docker_compose_dir
        )

        for varname in envfile_vars.difference(set(dockercompose_vars.keys())):
            if varname in ignored_varnames:
                continue

            if varname in envfile_vars:
                problems.append(
                    f'Envvar "{varname}" is defined in the .env file, but not found in any docker-compose file.'
                )

            if varname in dockercompose_vars:
                problems.append(
                    f'Envvar "{varname}" is used in the docker-compose file(s) {dockercompose_vars[varname]}, but not defined in the .env file.'
                )

    if args.k8s_environments_dir:
        k8s_vars = get_env_varnames_from_k8s_environments(args.k8s_environments_dir)
        k8s_environments = [
            path.name for path in Path(args.k8s_environments_dir).iterdir()
        ]

        for varname in envfile_vars.difference(set(k8s_vars.keys())):
            if varname in ignored_varnames:
                continue

            if varname in envfile_vars:
                problems.append(
                    f'Envvar "{varname}" is defined in the .env file, but not found in the any k8s configuration(s) and secret(s).'
                )

            if varname in k8s_vars:
                problems.append(
                    f'Envvar "{varname}" is used in the k8s configuration(s) and secret(s) {k8s_vars[varname]}, but not defined in the .env file.'
                )

        for varname, occurrences in k8s_vars.items():
            if varname in ignored_varnames:
                continue

            for environment in occurrences:
                if environment not in k8s_environments:

                    problems.append(
                        f'Envvar "{varname}" should be in all k8s environments, but missing not found neither in configuration or secrets of "{environment}".'
                    )

    if len(problems) == 0:
        print("All envvars are ok.")
        exit(0)
    else:
        for problem in sorted(problems):
            print(problem)

        print(
            f"Some envvars are not passed properly, {len(problems)} problem(s) found."
        )
        exit(1)


# scripts/create_storage_bucket.py

import boto3

# TODO: add cli params
# TODO: add non-delete bucket policy


def load_env_file():
    """Read env file and return a dict with the variables"""

    environment = {}
    with open("../.env") as f:
        for line in f:
            if line.strip():
                splitted = line.rstrip().split("=", maxsplit=1)
                if len(splitted) == 2:
                    environment[splitted[0]] = splitted[1]

    return environment


def _get_s3_bucket(env, bucket_name):
    """Get S3 Bucket according to the env variable
    STORAGE_BUCKET_NAME"""

    # Create session
    session = boto3.Session(
        aws_access_key_id=env["STORAGE_ACCESS_KEY_ID"],
        aws_secret_access_key=env["STORAGE_SECRET_ACCESS_KEY"],
        region_name=env["STORAGE_REGION_NAME"],
    )

    # Get the bucket objects
    s3 = session.resource("s3", endpoint_url=env["STORAGE_ENDPOINT_URL"])
    return s3.Bucket(bucket_name)


def _get_s3_client(env):

    s3_client = boto3.client(
        "s3",
        region_name=env["STORAGE_REGION_NAME"],
        aws_access_key_id=env["STORAGE_ACCESS_KEY_ID"],
        aws_secret_access_key=env["STORAGE_SECRET_ACCESS_KEY"],
        endpoint_url=env["STORAGE_ENDPOINT_URL"],
    )
    return s3_client


def _create_bucket(env, bucket_name):
    client = _get_s3_client(env)

    client.create_bucket(
        Bucket=bucket_name,
    )


def _enable_versioning(env, bucket_name):
    client = _get_s3_client(env)

    client.put_bucket_versioning(
        Bucket=bucket_name, VersioningConfiguration={"Status": "Enabled"}
    )


def _print_access_control_list(env, bucket_name):
    # Retrieve a bucket's ACL
    s3 = _get_s3_client(env)
    result = s3.get_bucket_acl(Bucket=bucket_name)
    from pprint import pprint

    pprint(result)


env = load_env_file()

# _print_access_control_list(env, 'qfieldcloud-test')
# _create_bucket(env, 'qfieldcloud-dev')

# bucket.objects.all().delete()
# bucket.object_versions.all().delete()

# client.delete_bucket(
#    Bucket='test-bucket',
#     )


# docker-app/qfieldcloud/authentication/migrations/0001_initial.py

# Generated by Django 3.2.7 on 2021-10-01 22:01

import django.db.models.deletion
import qfieldcloud.authentication.models
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name="AuthToken",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "key",
                    models.CharField(
                        db_index=True,
                        default=qfieldcloud.authentication.models.generate_token_key,
                        max_length=300,
                        unique=True,
                        verbose_name="Token",
                    ),
                ),
                (
                    "created_at",
                    models.DateTimeField(auto_now_add=True, verbose_name="Created at"),
                ),
                (
                    "expires_at",
                    models.DateTimeField(
                        default=qfieldcloud.authentication.models.generate_token_expires_at,
                        verbose_name="Expires at",
                    ),
                ),
                (
                    "updated_at",
                    models.DateTimeField(auto_now=True, verbose_name="Updated at"),
                ),
                (
                    "last_used_at",
                    models.DateTimeField(null=True, verbose_name="Last used at"),
                ),
                ("user_agent", models.TextField(blank=True, verbose_name="User-Agent")),
                (
                    "client_type",
                    models.CharField(
                        choices=[
                            ("browser", "Browser"),
                            ("cli", "Command line interface"),
                            ("sdk", "SDK"),
                            ("qfield", "QField"),
                            ("qfieldsync", "QFieldSync"),
                            ("unknown", "Unknown"),
                        ],
                        default="unknown",
                        max_length=32,
                    ),
                ),
                (
                    "user",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="auth_tokens",
                        to=settings.AUTH_USER_MODEL,
                        verbose_name="User",
                    ),
                ),
            ],
            options={
                "verbose_name": "Token",
                "verbose_name_plural": "Tokens",
                "ordering": ("-created_at",),
            },
        ),
    ]


# docker-app/qfieldcloud/authentication/migrations/0002_alter_authtoken_client_type.py

# Generated by Django 3.2.12 on 2022-03-09 15:58

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("authentication", "0001_initial"),
    ]

    operations = [
        migrations.AlterField(
            model_name="authtoken",
            name="client_type",
            field=models.CharField(
                choices=[
                    ("browser", "Browser"),
                    ("cli", "Command line interface"),
                    ("sdk", "SDK"),
                    ("qfield", "QField"),
                    ("qfieldsync", "QFieldSync"),
                    ("worker", "Worker"),
                    ("unknown", "Unknown"),
                ],
                default="unknown",
                max_length=32,
            ),
        ),
    ]


# docker-app/qfieldcloud/authentication/tests/test_authentication.py

import logging

from django.utils.timezone import datetime, now
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.models import User
from qfieldcloud.core.tests.utils import setup_subscription_plans
from rest_framework.test import APITransactionTestCase

logging.disable(logging.CRITICAL)


class QfcTestCase(APITransactionTestCase):
    def setUp(self):
        setup_subscription_plans()

        # Create a user
        self.user1 = User.objects.create_user(username="user1", password="abc123")

    def assertTokenMatch(self, token, payload):
        expires_at = payload.pop("expires_at")
        avatar_url = payload.pop("avatar_url")
        self.assertDictEqual(
            payload,
            {
                "token": token.key,
                # "expires_at": tokens[0].expires_at.isoformat(),
                "username": token.user.username,
                "email": "",
                "first_name": "",
                "full_name": "",
                "last_name": "",
                "user_type": "1",
            },
        )
        self.assertTrue(datetime.fromisoformat(expires_at) == token.expires_at)
        self.assertTrue(datetime.fromisoformat(expires_at) > now())
        self.assertTrue(avatar_url is None or avatar_url.startswith("http"))
        self.assertTrue(
            avatar_url is None
            or avatar_url.endswith(
                f"/api/v1/files/public/users/{token.user.username}/avatar.svg"
            )
        )

    def login(self, username, password, user_agent=""):
        response = self.client.post(
            "/api/v1/auth/login/",
            {
                "username": username,
                "password": password,
            },
            HTTP_USER_AGENT=user_agent,
        )

        self.assertEqual(response.status_code, 200)

        return response

    def test_login_logout(self):
        response = self.login("user1", "abc123")
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertEquals(len(tokens), 1)
        self.assertTokenMatch(tokens[0], response.json())
        self.assertGreater(tokens[0].expires_at, now())

        # set auth token
        self.client.credentials(HTTP_AUTHORIZATION="Token " + tokens[0].key)

        # logout
        response = self.client.post("/api/v1/auth/logout/")
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertEqual(response.status_code, 200)

        self.assertEquals(len(tokens), 1)
        self.assertLess(tokens[0].expires_at, now())

    def test_multiple_logins(self):
        # first single active token login
        response = self.login("user1", "abc123", "Mozilla/5.0 QGIS/32203")
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertEquals(len(tokens), 1)
        self.assertTokenMatch(tokens[0], response.json())

        # second single active token login
        response = self.login("user1", "abc123", "Mozilla/5.0 QGIS/32203")
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertEquals(len(tokens), 2)
        self.assertTokenMatch(tokens[0], response.json())
        self.assertNotEqual(tokens[0], tokens[1])
        self.assertGreater(tokens[0].expires_at, now())
        self.assertLess(tokens[1].expires_at, now())

        # first single active token login
        response = self.login("user1", "abc123", "sdk|py|dev python-requests|2.26.0")
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertEquals(len(tokens), 3)
        self.assertTokenMatch(tokens[0], response.json())

        # second single active token login
        response = self.login("user1", "abc123", "sdk|py|dev python-requests|2.26.0")
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertEquals(len(tokens), 4)
        self.assertTokenMatch(tokens[0], response.json())
        self.assertNotEqual(tokens[0], tokens[1])
        self.assertGreater(tokens[0].expires_at, now())
        self.assertGreater(tokens[1].expires_at, now())

    def test_client_type(self):
        # QFIELDSYNC login
        response = self.login("user1", "abc123", "Mozilla/5.0 QGIS/32203")
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertTokenMatch(tokens[0], response.json())
        self.assertEqual(tokens[0].client_type, AuthToken.ClientType.QFIELDSYNC)

        response = self.login(
            "user1", "abc123", "Mozilla/5.0 QGIS/32400/Ubuntu 20.04.4 LTS"
        )
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertTokenMatch(tokens[0], response.json())
        self.assertEqual(tokens[0].client_type, AuthToken.ClientType.QFIELDSYNC)

        # SDK login
        response = self.login("user1", "abc123", "sdk|py|dev python-requests|2.26.0")
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertTokenMatch(tokens[0], response.json())
        self.assertEqual(tokens[0].client_type, AuthToken.ClientType.SDK)

        # BROWSER login
        response = self.login(
            "user1",
            "abc123",
            "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:97.0) Gecko/20100101 Firefox/97.0",
        )
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertTokenMatch(tokens[0], response.json())
        self.assertEqual(tokens[0].client_type, AuthToken.ClientType.BROWSER)

        response = self.login(
            "user1",
            "abc123",
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.105 Safari/537.36",
        )
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertTokenMatch(tokens[0], response.json())
        self.assertEqual(tokens[0].client_type, AuthToken.ClientType.BROWSER)

        # QFIELD login
        response = self.login(
            "user1",
            "abc123",
            "qfield|1.0.0|local - dev|3515ce8cba0f0e0abb92e06bf30a00531810656f| QGIS/31900",
        )
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertTokenMatch(tokens[0], response.json())
        self.assertEqual(tokens[0].client_type, AuthToken.ClientType.QFIELD)

        # UNKNOWN login
        response = self.login("user1", "abc123", "Слава Україні!")
        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertTokenMatch(tokens[0], response.json())
        self.assertEqual(tokens[0].client_type, AuthToken.ClientType.UNKNOWN)

    def test_last_used_at(self):
        response = self.login("user1", "abc123")

        tokens = self.user1.auth_tokens.order_by("-created_at").all()

        self.assertEquals(len(tokens), 1)
        self.assertTokenMatch(tokens[0], response.json())
        self.assertIsNone(tokens[0].last_used_at)

        # set auth token
        self.client.credentials(HTTP_AUTHORIZATION="Token " + tokens[0].key)

        # first token usage
        response = self.client.get(f"/api/v1/users/{self.user1.username}/")

        self.assertEqual(response.status_code, 200)

        tokens = self.user1.auth_tokens.order_by("-created_at").all()
        first_used_at = tokens[0].last_used_at

        self.assertEquals(len(tokens), 1)

        # second token usage
        response = self.client.get(f"/api/v1/users/{self.user1.username}/")

        self.assertEqual(response.status_code, 200)

        tokens = self.user1.auth_tokens.order_by("-created_at").all()
        second_used_at = tokens[0].last_used_at

        self.assertEquals(len(tokens), 1)
        self.assertLess(first_used_at, second_used_at)


# docker-app/qfieldcloud/notifs/tests/test_notifs.py

from datetime import timedelta

from django.core.management import call_command
from django.db import close_old_connections
from django.test import TestCase
from django.utils import timezone
from django_cron.management.commands import runcrons
from django_currentuser.middleware import _set_current_user
from notifications.models import Notification
from notifications.signals import notify
from qfieldcloud.core.models import (
    Organization,
    OrganizationMember,
    Project,
    Team,
    User,
    UserAccount,
)
from qfieldcloud.core.tests.utils import setup_subscription_plans


class QfcTestCase(TestCase):
    """
    This tests that the notifications are sent
    """

    @classmethod
    def setUpClass(cls):
        # remove this once https://github.com/Tivix/django-cron/pull/176 is merged
        cls._close_old_connections = close_old_connections

        def noop():
            pass

        runcrons.close_connection = noop

    @classmethod
    def tearDownClass(cls):
        # restore above monkeypatch
        runcrons.close_connection = cls._close_old_connections

    def assertNotifs(self, expected_count, filter=None):
        if filter is None:
            filter = {}
        notifications = Notification.objects.filter(**filter)
        actual_count = notifications.count()
        if actual_count != expected_count:
            details = "\n".join(f"- {n}" for n in notifications)
            raise AssertionError(
                f"Expected {expected_count}, but got {actual_count} for {filter}\nDetails:\n{details}"
            )

    def setUp(self):
        setup_subscription_plans()

        # Without current user, no notifications are sent
        # (one day, maybe we could create them as AnonymousUser, see https://github.com/django-notifications/django-notifications/issues/13)
        _set_current_user(None)

        self.user1 = User.objects.create_user(username="user1", password="abc123")
        self.user2 = User.objects.create_user(username="user2", password="abc123")
        self.user3 = User.objects.create_user(username="user3", password="abc123")
        self.otheruser = User.objects.create_user(
            username="otheruser", password="abc123"
        )

    def test_no_self_notifications(self):
        # No notification to the actor of the action
        # (at some point, we could create the notif but set it as read)
        _set_current_user(self.user1)
        Organization.objects.create(username="org2", organization_owner=self.user1)
        self.assertNotifs(0, {"recipient": self.user1})

    def test_organization(self):

        _set_current_user(self.otheruser)

        # Org owner is notified of creation
        org1 = Organization.objects.create(
            username="org1", organization_owner=self.user1
        )
        self.assertNotifs(1, {"recipient": self.user1})

        # Org owner is notified of deletion
        org1.delete()
        self.assertNotifs(2, {"recipient": self.user1})

    def test_organization_members(self):

        org1 = Organization.objects.create(
            username="org1", organization_owner=self.user1
        )

        _set_current_user(self.otheruser)

        # Set user2, user3 as member of organization1
        OrganizationMember.objects.create(
            organization=org1,
            member=self.user2,
            role=OrganizationMember.Roles.MEMBER,
        )
        memb2 = OrganizationMember.objects.create(
            organization=org1,
            member=self.user3,
            role=OrganizationMember.Roles.MEMBER,
        )
        self.assertNotifs(2, {"recipient": self.user1})
        self.assertNotifs(2, {"recipient": self.user2})
        self.assertNotifs(1, {"recipient": self.user3})

        memb2.delete()
        self.assertNotifs(3, {"recipient": self.user1})
        self.assertNotifs(3, {"recipient": self.user2})
        self.assertNotifs(2, {"recipient": self.user3})

        org1.delete()
        self.assertNotifs(5, {"recipient": self.user1})
        self.assertNotifs(5, {"recipient": self.user2})
        self.assertNotifs(2, {"recipient": self.user3})

    def test_team_members(self):

        org1 = Organization.objects.create(
            username="org1", organization_owner=self.user1
        )
        memb2 = org1.members.create(member=self.user2)
        org1.members.create(member=self.user3)
        t1 = org1.teams.create(username="t1")

        _set_current_user(self.otheruser)

        # Set user2, user3 as members of team1
        t1.members.create(member=self.user2)
        t1.members.create(member=self.user3)

        self.assertNotifs(2, {"recipient": self.user1})
        self.assertNotifs(2, {"recipient": self.user2})
        self.assertNotifs(1, {"recipient": self.user3})

        memb2.delete()
        self.assertNotifs(4, {"recipient": self.user1})
        self.assertNotifs(4, {"recipient": self.user2})
        self.assertNotifs(3, {"recipient": self.user3})

        t1.delete()
        self.assertNotifs(6, {"recipient": self.user1})
        self.assertNotifs(4, {"recipient": self.user2})
        self.assertNotifs(5, {"recipient": self.user3})

    def test_projects(self):

        org1 = Organization.objects.create(
            username="org1", organization_owner=self.user1
        )
        org1.members.create(member=self.user2)
        t1 = Team.objects.create(username="t1", team_organization=org1)
        t1.members.create(member=self.user2)

        _set_current_user(self.otheruser)

        # create a project
        p1 = Project.objects.create(name="p1", owner=org1, is_public=True)
        self.assertNotifs(1, {"recipient": self.user1})
        self.assertNotifs(0, {"recipient": self.user2})
        self.assertNotifs(0, {"recipient": self.user3})

        # add a collaborator (team)
        p1.collaborators.create(collaborator=t1)
        self.assertNotifs(2, {"recipient": self.user1})
        self.assertNotifs(1, {"recipient": self.user2})
        self.assertNotifs(0, {"recipient": self.user3})

        # add a collaborator (user)
        org1.members.create(member=self.user3)
        p1.collaborators.create(collaborator=self.user3)
        self.assertNotifs(4, {"recipient": self.user1})
        self.assertNotifs(3, {"recipient": self.user2})
        self.assertNotifs(2, {"recipient": self.user3})

    def test_cron(self):
        # Ensuring cron works

        user1 = self.user1
        user1.email = "test@example.com"
        user1.save()

        old = timezone.now() - timedelta(minutes=65)
        now = timezone.now()

        notify.send(
            user1,
            verb="tests_old",
            action_object=user1,
            recipient=[user1],
            timestamp=old,
        )
        notify.send(
            user1,
            verb="tests_now",
            action_object=user1,
            recipient=[user1],
            timestamp=now,
        )

        # When disabled, no cron job sends notifications
        user1.useraccount.notifs_frequency = UserAccount.NOTIFS_DISABLED
        user1.useraccount.save()
        self.assertNotifs(2, {"emailed": False})
        self.assertNotifs(0, {"emailed": True})
        call_command("runcrons", "--force")
        self.assertNotifs(2, {"emailed": False})
        self.assertNotifs(0, {"emailed": True})

        # When daily, we still don't send, as notification isn't old enough
        user1.useraccount.notifs_frequency = UserAccount.NOTIFS_DAILY
        user1.useraccount.save()
        self.assertNotifs(2, {"emailed": False})
        self.assertNotifs(0, {"emailed": True})
        call_command("runcrons", "--force")
        self.assertNotifs(2, {"emailed": False})
        self.assertNotifs(0, {"emailed": True})

        # When hourly, we send all notifs, since we have one that is old enough
        user1.useraccount.notifs_frequency = UserAccount.NOTIFS_HOURLY
        user1.useraccount.save()
        self.assertNotifs(2, {"emailed": False})
        self.assertNotifs(0, {"emailed": True})
        call_command("runcrons", "--force")
        self.assertNotifs(0, {"emailed": False})
        self.assertNotifs(2, {"emailed": True})


# docker-app/qfieldcloud/subscription/migrations/0001_initial.py

# Generated by Django 3.2.11 on 2022-02-08 20:59

import datetime

import django.core.validators
import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ("core", "0051_auto_20211125_0444"),
    ]

    operations = [
        migrations.CreateModel(
            name="Plan",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("code", models.CharField(max_length=100, unique=True)),
                (
                    "user_type",
                    models.PositiveSmallIntegerField(
                        choices=[(1, "User"), (2, "Organization")], default=1
                    ),
                ),
                (
                    "ordering",
                    models.PositiveIntegerField(
                        default=0,
                        help_text='Relative ordering of the record. Lower values have higher priority (will be first in the list). Records with same ordering will be sorted by "Display name" and "Code". Please set with gaps for different records for easy reordering (e.g. 5, 10, 15, but not 5, 6, 7).',
                    ),
                ),
                ("display_name", models.CharField(max_length=100)),
                ("storage_mb", models.PositiveIntegerField(default=10)),
                ("storage_keep_versions", models.PositiveIntegerField(default=10)),
                ("job_minutes", models.PositiveIntegerField(default=10)),
                ("can_add_storage", models.BooleanField(default=False)),
                ("can_add_job_minutes", models.BooleanField(default=False)),
                ("is_external_db_supported", models.BooleanField(default=False)),
                (
                    "can_configure_repackaging_cache_expire",
                    models.BooleanField(default=False),
                ),
                (
                    "min_repackaging_cache_expire",
                    models.DurationField(
                        default=datetime.timedelta(seconds=3600),
                        validators=[
                            django.core.validators.MinValueValidator(
                                datetime.timedelta(seconds=60)
                            )
                        ],
                    ),
                ),
                (
                    "synchronizations_per_months",
                    models.PositiveIntegerField(default=30),
                ),
                ("is_public", models.BooleanField(default=False)),
                ("is_default", models.BooleanField(default=False)),
                ("is_premium", models.BooleanField(default=False)),
                (
                    "max_organization_members",
                    models.IntegerField(
                        default=-1,
                        help_text="Maximum organization members allowed. Set -1 to allow unlimited organization members.",
                    ),
                ),
                (
                    "max_premium_collaborators_per_private_project",
                    models.IntegerField(
                        default=-1,
                        help_text="Maximum premium collaborators per private project. Set -1 to allow unlimited project collaborators.",
                    ),
                ),
            ],
            options={
                "ordering": (
                    "ordering",
                    "display_name",
                    "code",
                )
            },
        ),
        migrations.CreateModel(
            name="ExtraPackageType",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("code", models.CharField(max_length=100, unique=True)),
                ("display_name", models.CharField(max_length=100)),
                ("is_public", models.BooleanField(default=False)),
            ],
        ),
        migrations.CreateModel(
            name="ExtraPackageTypeJobMinutes",
            fields=[
                (
                    "extrapackagetype_ptr",
                    models.OneToOneField(
                        auto_created=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        parent_link=True,
                        primary_key=True,
                        serialize=False,
                        to="subscription.extrapackagetype",
                    ),
                ),
                ("minutes", models.PositiveIntegerField()),
            ],
            bases=("subscription.extrapackagetype",),
        ),
        migrations.CreateModel(
            name="ExtraPackageTypeStorage",
            fields=[
                (
                    "extrapackagetype_ptr",
                    models.OneToOneField(
                        auto_created=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        parent_link=True,
                        primary_key=True,
                        serialize=False,
                        to="subscription.extrapackagetype",
                    ),
                ),
                ("megabytes", models.PositiveIntegerField()),
            ],
            bases=("subscription.extrapackagetype",),
        ),
        migrations.CreateModel(
            name="ExtraPackage",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("start_date", models.DateField()),
                ("end_date", models.DateField(blank=True, null=True)),
                (
                    "quantity",
                    models.PositiveIntegerField(
                        validators=[
                            django.core.validators.MinValueValidator(1),
                            django.core.validators.MaxValueValidator(25),
                        ],
                    ),
                ),
                (
                    "type",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="packages",
                        to="subscription.extrapackagetype",
                    ),
                ),
                (
                    "account",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="extra_packages",
                        to="core.useraccount",
                    ),
                ),
            ],
        ),
    ]


# docker-app/qfieldcloud/subscription/migrations/0002_populate_plans.py

from datetime import timedelta

from django.core.management.color import no_style
from django.db import connection, migrations


def populate_plans(apps, schema_editor):
    Plan = apps.get_model("subscription", "Plan")

    Plan.objects.create(
        id=1,  # DO NOT CHANGE! Must match legacy UserAccount.TYPE_COMMUNITY
        code="community",
        user_type=1,
        ordering=1100,
        display_name="community",
        storage_mb=10000,
        storage_keep_versions=10,
        job_minutes=10000,
        can_add_storage=False,
        can_add_job_minutes=False,
        is_external_db_supported=False,
        can_configure_repackaging_cache_expire=False,
        min_repackaging_cache_expire=timedelta(minutes=60),
        synchronizations_per_months=30,
        max_organization_members=-1,
        is_public=True,
        is_default=True,
    )

    Plan.objects.create(
        id=2,
        code="organization",
        user_type=2,
        ordering=5100,
        display_name="organization",
        storage_mb=5000,
        storage_keep_versions=10,
        job_minutes=100,  # TODO: QF-234 says per should be per user !
        can_add_storage=False,
        can_add_job_minutes=False,
        is_external_db_supported=True,
        can_configure_repackaging_cache_expire=True,
        min_repackaging_cache_expire=timedelta(minutes=1),
        synchronizations_per_months=30,
        max_organization_members=-1,
        is_public=True,
        is_default=True,
    )

    # Reset the Postgres sequences due to hardcoded ids
    sequence_sql = connection.ops.sequence_reset_sql(no_style(), [Plan])
    with connection.cursor() as cursor:
        for sql in sequence_sql:
            cursor.execute(sql)

    ExtraPackageTypeStorage = apps.get_model("subscription", "ExtraPackageTypeStorage")
    ExtraPackageTypeStorage.objects.create(
        code="storage_medium",
        display_name="Medium +1000MB",
        is_public=True,
        megabytes=1000,
    )

    ExtraPackageTypeJobMinutes = apps.get_model(
        "subscription", "ExtraPackageTypeJobMinutes"
    )
    ExtraPackageTypeJobMinutes.objects.create(
        code="minutes_medium",
        display_name="minutes_medium",
        is_public=True,
        minutes=30,
    )


class Migration(migrations.Migration):

    dependencies = [
        ("subscription", "0001_initial"),
    ]

    operations = [
        migrations.RunPython(populate_plans, migrations.RunPython.noop),
    ]


# docker-app/qfieldcloud/subscription/tests/test_external_db.py

import io
import json
from pathlib import Path
from time import sleep

import psycopg2
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.geodb_utils import delete_db_and_role
from qfieldcloud.core.models import Delta, Geodb, Job, Project, User
from qfieldcloud.core.tests.utils import setup_subscription_plans
from rest_framework import status
from rest_framework.test import APITransactionTestCase

from ..models import Plan

DATA_FOLDER = Path(__file__).parent / "data"


class QfcTestCase(APITransactionTestCase):
    def _login(self, user):
        token = AuthToken.objects.get_or_create(user=user)[0]
        self.client.credentials(HTTP_AUTHORIZATION=f"Token {token}")

    def _wait(self, project, job_type, expected_result=Job.Status.FINISHED):
        # print(f"Waiting for {job_type} status {expected_result}", end="")
        for _ in range(30):
            job = (
                Job.objects.filter(project=project, type=job_type)
                .exclude(
                    status__in=[
                        Job.Status.PENDING,
                        Job.Status.QUEUED,
                        Job.Status.STARTED,
                    ]
                )
                .last()
            )
            if job:
                if job.status == expected_result:
                    # print(" done")
                    break
                else:
                    print(f" got unexpected result ! ({job.status})")
                    print("~~~~~~~~~~~~~~~~~~~~~~")
                    print(job.feedback)
                    print("~~~~~~~~~~~~~~~~~~~~~~")
                    raise Exception("Unexpected result failed !")
            # print(".", end="")
            sleep(1)
        else:
            raise Exception("Processing did no finish, did the worker hang ?")

    def _get_delta_file_with_project_id(self, project, delta_filename):
        """Retrieves a delta json file with the project id replaced by the project.id"""
        with open(delta_filename, "r") as f:
            deltafile = json.load(f)
            deltafile["project"] = str(project.id)
            json_str = json.dumps(deltafile)
            return io.StringIO(json_str)

    def setUp(self):
        setup_subscription_plans()

    def test_is_external_db_supported(self):
        """This tests is_external_db_supported property of accounts types"""

        u1 = User.objects.create(username="u1")
        self._login(u1)

        # Create a project with a writable remote DB

        p1 = Project.objects.create(name="p1", owner=u1)

        delete_db_and_role("test", "usr1")
        Geodb.objects.create(
            dbname="test",
            user=u1,
            username="usr1",
            password="pwd",
            hostname="geodb",
            port=5432,
        )
        conn = psycopg2.connect(
            dbname="test",
            user="usr1",
            password="pwd",
            host="geodb",
            port=5432,
        )
        conn.cursor().execute("CREATE TABLE point (id integer PRIMARY KEY, name text)")
        conn.commit()

        # Upload the project file and ensure it loaded

        response = self.client.post(
            f"/api/v1/files/{p1.id}/project.qgs/",
            {"file": open(DATA_FOLDER / "project_pgservice.qgs", "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self._wait(p1, Job.Type.PROCESS_PROJECTFILE)

        # Ensure we start without delta

        self.assertEqual(Delta.objects.filter(project=p1).count(), 0)

        # When external db supported, we can apply deltas

        Plan.objects.all().update(is_external_db_supported=True)
        response = self.client.post(
            f"/api/v1/deltas/{p1.id}/",
            {
                "file": self._get_delta_file_with_project_id(
                    p1, DATA_FOLDER / "project_pgservice_delta_1.json"
                )
            },
            format="multipart",
        )
        self.assertEqual(Delta.objects.filter(project=p1).count(), 1)
        self.assertEqual(
            Delta.objects.filter(project=p1).latest("created_at").last_status,
            Delta.Status.PENDING,
        )
        self._wait(p1, Job.Type.DELTA_APPLY)
        self.assertEqual(
            Delta.objects.filter(project=p1).latest("created_at").last_status,
            Delta.Status.APPLIED,
        )

        # When external db is NOT supported, we can NOT apply deltas

        Plan.objects.all().update(is_external_db_supported=False)
        response = self.client.post(
            f"/api/v1/deltas/{p1.id}/",
            {
                "file": self._get_delta_file_with_project_id(
                    p1, DATA_FOLDER / "project_pgservice_delta_2.json"
                )
            },
            format="multipart",
        )
        self.assertEqual(Delta.objects.filter(project=p1).count(), 2)
        print("-------------------")
        for delta in Delta.objects.filter(project=p1):
            print(delta)
        self.assertEqual(
            Delta.objects.filter(project=p1).latest("created_at").last_status,
            Delta.Status.UNPERMITTED,
        )


# docker-app/qfieldcloud/subscription/tests/test_organization.py

import logging

from django.core.exceptions import ValidationError
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.models import Organization, OrganizationMember, User
from qfieldcloud.core.tests.utils import setup_subscription_plans
from rest_framework.test import APITransactionTestCase

from ..models import Plan

logging.disable(logging.CRITICAL)


class QfcTestCase(APITransactionTestCase):
    def _login(self, user):
        token = AuthToken.objects.get_or_create(user=user)[0]
        self.client.credentials(HTTP_AUTHORIZATION=f"Token {token}")

    def setUp(self):
        setup_subscription_plans()

    def test_max_organization_members(self):
        """This tests quotas"""

        u1 = User.objects.create(username="u1")
        u2 = User.objects.create(username="u2")
        u3 = User.objects.create(username="u3")
        u4 = User.objects.create(username="u4")
        self._login(u1)

        o1 = Organization.objects.create(username="o1", organization_owner=u1)
        unlimited_plan = Plan.objects.create(
            code="max_organization_members_minus1",
            user_type=Plan.UserType.ORGANIZATION,
            max_organization_members=-1,
        )
        limited_plan = Plan.objects.create(
            code="max_organization_members_plus1",
            user_type=Plan.UserType.ORGANIZATION,
            max_organization_members=1,
        )

        o1.useraccount.plan = unlimited_plan
        o1.useraccount.save()

        OrganizationMember.objects.create(member=u2, organization=o1)
        OrganizationMember.objects.create(member=u3, organization=o1)

        o1.useraccount.plan = limited_plan
        o1.useraccount.save()

        with self.assertRaises(ValidationError):
            OrganizationMember.objects.create(member=u4, organization=o1)

        o2 = Organization.objects.create(username="o2", organization_owner=u1)
        o2.useraccount.plan = limited_plan
        o2.useraccount.save()

        OrganizationMember.objects.create(member=u2, organization=o2)

        with self.assertRaises(ValidationError):
            OrganizationMember.objects.create(member=u3, organization=o2)


# docker-app/qfieldcloud/subscription/tests/test_storage_quota.py

import io
import logging
import os
from datetime import date, timedelta

from django_currentuser.middleware import _set_current_user
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core import utils
from qfieldcloud.core.models import Project, User
from qfieldcloud.core.tests.utils import setup_subscription_plans
from qfieldcloud.core.utils import list_versions
from qfieldcloud.core.utils2.storage import delete_file_version
from rest_framework import status
from rest_framework.test import APITransactionTestCase

from ..models import ExtraPackage, ExtraPackageTypeStorage, Plan

logging.disable(logging.CRITICAL)


class QfcTestCase(APITransactionTestCase):
    def _login(self, user):
        token = AuthToken.objects.get_or_create(user=user)[0]
        self.client.credentials(HTTP_AUTHORIZATION=f"Token {token}")

    def _make_file(self, mb):
        """Helper that returns a file of given size in megabytes"""
        return io.BytesIO(os.urandom(1000 * int(mb * 1000)))

    def setUp(self):
        setup_subscription_plans()

    def test_storage_quota_calculation(self):
        """This tests quotas"""

        u1 = User.objects.create(username="u1")
        p1 = Project.objects.create(name="p1", owner=u1)

        # NOTE since we do not login via the API, the django_currentuser module is unable to set
        # the current user from the request object. Therefore the user set from another test
        # remains in the internal django_currentuser cache and causes the test to fail. So we have
        # to manually reset the current user to None.
        _set_current_user(None)

        default = Plan.get_or_create_default()

        # Initially, we have the default account type storage
        self.assertEqual(u1.useraccount.storage_quota_left_mb, default.storage_mb)
        self.assertEqual(u1.useraccount.storage_quota_used_mb, 0)
        self.assertEqual(u1.useraccount.storage_quota_total_mb, default.storage_mb)
        self.assertEqual(u1.useraccount.storage_quota_used_perc, 0)

        # Changing account type changes the quota
        plan_1mb = Plan.objects.create(code="plan_1mb", storage_mb=1)
        u1.useraccount.plan = plan_1mb
        u1.useraccount.save()
        self.assertEqual(u1.useraccount.storage_quota_left_mb, 1)
        self.assertEqual(u1.useraccount.storage_quota_used_mb, 0)
        self.assertEqual(u1.useraccount.storage_quota_total_mb, 1)
        self.assertEqual(u1.useraccount.storage_quota_used_perc, 0)

        # Adding an extra package increases the quota
        extra_2mb = ExtraPackageTypeStorage.objects.create(
            code="extra_2mb", megabytes=2
        )
        ExtraPackage.objects.create(
            account=u1.useraccount,
            type=extra_2mb,
            quantity=1,
            start_date=date.today() - timedelta(days=3),
            end_date=date.today() + timedelta(days=3),
        )
        self.assertEqual(u1.useraccount.storage_quota_left_mb, 3)
        self.assertEqual(u1.useraccount.storage_quota_used_mb, 0)
        self.assertEqual(u1.useraccount.storage_quota_total_mb, 3)
        self.assertEqual(u1.useraccount.storage_quota_used_perc, 0)

        # Adding an obsolete package does not count
        ExtraPackage.objects.create(
            account=u1.useraccount,
            type=extra_2mb,
            quantity=1,
            start_date=date.today() - timedelta(days=3),
            end_date=date.today() - timedelta(days=2),
        )
        self.assertEqual(u1.useraccount.storage_quota_left_mb, 3)
        self.assertEqual(u1.useraccount.storage_quota_used_mb, 0)
        self.assertEqual(u1.useraccount.storage_quota_total_mb, 3)
        self.assertEqual(u1.useraccount.storage_quota_used_perc, 0)

        # Adding a future package does not count
        ExtraPackage.objects.create(
            account=u1.useraccount,
            type=extra_2mb,
            quantity=1,
            start_date=date.today() + timedelta(days=2),
            end_date=date.today() + timedelta(days=3),
        )
        self.assertEqual(u1.useraccount.storage_quota_left_mb, 3)
        self.assertEqual(u1.useraccount.storage_quota_used_mb, 0)
        self.assertEqual(u1.useraccount.storage_quota_total_mb, 3)
        self.assertEqual(u1.useraccount.storage_quota_used_perc, 0)

        # Adding a timeless package increases the quota
        ExtraPackage.objects.create(
            account=u1.useraccount,
            type=extra_2mb,
            quantity=1,
            start_date=date.today() - timedelta(days=3),
            end_date=None,
        )
        self.assertEqual(u1.useraccount.storage_quota_left_mb, 5)
        self.assertEqual(u1.useraccount.storage_quota_used_mb, 0)
        self.assertEqual(u1.useraccount.storage_quota_total_mb, 5)
        self.assertEqual(u1.useraccount.storage_quota_used_perc, 0)

        # Uploading a file decreases the quota
        storage_path = f"projects/{p1.id}/files/test.data"
        bucket = utils.get_s3_bucket()
        bucket.upload_fileobj(self._make_file(mb=1), storage_path)
        p1.save(recompute_storage=True)
        self.assertEqual(u1.useraccount.storage_quota_left_mb, 4)
        self.assertEqual(u1.useraccount.storage_quota_used_mb, 1)
        self.assertEqual(u1.useraccount.storage_quota_total_mb, 5)
        self.assertEqual(u1.useraccount.storage_quota_used_perc, 20)

        # Uploading a new version decreases the quota
        bucket = utils.get_s3_bucket()
        bucket.upload_fileobj(self._make_file(mb=1), storage_path)
        p1.save(recompute_storage=True)
        self.assertEqual(u1.useraccount.storage_quota_left_mb, 3)
        self.assertEqual(u1.useraccount.storage_quota_used_mb, 2)
        self.assertEqual(u1.useraccount.storage_quota_total_mb, 5)
        self.assertEqual(u1.useraccount.storage_quota_used_perc, 40)

        # Deleting a version increases the quota
        version = list(list_versions(bucket, storage_path))[0]
        delete_file_version(p1, "test.data", version.id)
        p1.save(recompute_storage=True)
        self.assertEqual(u1.useraccount.storage_quota_left_mb, 4)
        self.assertEqual(u1.useraccount.storage_quota_used_mb, 1)
        self.assertEqual(u1.useraccount.storage_quota_total_mb, 5)
        self.assertEqual(u1.useraccount.storage_quota_used_perc, 20)

    def test_api_enforces_storage_limit(self):
        u1 = User.objects.create(username="u1")
        p1 = Project.objects.create(name="p1", owner=u1)
        plan_1mb = Plan.objects.create(code="plan_1mb", storage_mb=1)
        u1.useraccount.plan = plan_1mb
        u1.useraccount.save()

        self._login(u1)

        apipath = f"/api/v1/files/{p1.id}/file.data/"

        # One file of 750kb is under quota of 1mb
        response = self.client.post(
            apipath, {"file": self._make_file(mb=0.75)}, format="multipart"
        )
        self.assertTrue(status.is_success(response.status_code))

        # A second file of 750kb is over quota of 1mb
        response = self.client.post(
            apipath, {"file": self._make_file(mb=0.75)}, format="multipart"
        )
        self.assertEqual(response.status_code, 402)

    def test_api_enforces_storage_limit_for_reparenting(self):
        plan_1mb = Plan.objects.create(code="plan_1mb", storage_mb=1)
        plan_2mb = Plan.objects.create(code="plan_2mb", storage_mb=2)
        extra_1mb = ExtraPackageTypeStorage.objects.create(
            code="extra_1mb", display_name="extra_1mb", megabytes=1
        )

        u1 = User.objects.create(username="u1")
        u1.useraccount.plan = plan_1mb
        u1.useraccount.save()

        u2 = User.objects.create(username="u2")
        u2.useraccount.plan = plan_2mb
        u2.useraccount.save()

        p1 = Project.objects.create(name="p1", owner=u2)

        self._login(u2)

        # User 2 uploads a 1.5mb file
        apipath = f"/api/v1/files/{p1.id}/file.data/"
        response = self.client.post(
            apipath, {"file": self._make_file(mb=1.5)}, format="multipart"
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertFalse(Project.objects.filter(owner=u1).exists())
        self.assertTrue(Project.objects.filter(owner=u2).exists())

        # Transfers to user1, but too much for his quota, so we refuse transfer
        apipath = f"/api/v1/projects/{p1.id}/"
        response = self.client.patch(apipath, {"owner": "u1"})
        self.assertEqual(response.status_code, 402)
        self.assertFalse(Project.objects.filter(owner=u1).exists())
        self.assertTrue(Project.objects.filter(owner=u2).exists())

        # User 1 buys a package, transfer now works
        ExtraPackage.objects.create(
            account=u1.useraccount,
            type=extra_1mb,
            quantity=1,
            start_date=date.today() - timedelta(days=3),
            end_date=None,
        )
        response = self.client.patch(apipath, {"owner": "u1"})
        self.assertTrue(status.is_success(response.status_code))
        self.assertTrue(Project.objects.filter(owner=u1).exists())
        self.assertFalse(Project.objects.filter(owner=u2).exists())


# docker-app/worker_wrapper/__init__.py

import django

django.setup()


# docker-app/worker_wrapper/wrapper.py

import json
import logging
import os
import sys
import tempfile
import traceback
import uuid
from datetime import timedelta
from pathlib import Path
from typing import Any, Dict, Iterable, List, Tuple

import docker
import qfieldcloud.core.utils2.storage
import requests
from django.conf import settings
from django.db import transaction
from django.forms.models import model_to_dict
from django.utils import timezone
from docker.models.containers import Container
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.models import (
    ApplyJob,
    ApplyJobDelta,
    Delta,
    Job,
    PackageJob,
    ProcessProjectfileJob,
    Secret,
)
from qfieldcloud.core.utils import get_qgis_project_file
from qfieldcloud.core.utils2 import storage

logger = logging.getLogger(__name__)

TIMEOUT_ERROR_EXIT_CODE = -1
QGIS_CONTAINER_NAME = os.environ.get("QGIS_CONTAINER_NAME", None)
QFIELDCLOUD_HOST = os.environ.get("QFIELDCLOUD_HOST", None)

assert QGIS_CONTAINER_NAME
assert QFIELDCLOUD_HOST


class QgisException(Exception):
    pass


class JobRun:
    container_timeout_secs = settings.WORKER_TIMEOUT_S
    job_class = Job
    command = []

    def __init__(self, job_id: str) -> None:
        try:
            self.job_id = job_id
            self.job = self.job_class.objects.select_related().get(id=job_id)
            self.shared_tempdir = Path(tempfile.mkdtemp(dir="/tmp"))
        except Exception as err:
            feedback = {}
            (_type, _value, tb) = sys.exc_info()
            feedback["error"] = str(err)
            feedback["error_origin"] = "worker_wrapper"
            feedback["error_stack"] = traceback.format_tb(tb)

            msg = "Uncaught exception when constructing a JobRun:\n"
            msg += json.dumps(msg, indent=2, sort_keys=True)

            if self.job:
                self.job.status = Job.Status.FAILED
                self.job.feedback = feedback
                self.job.save()
                logger.exception(msg, exc_info=err)
            else:
                logger.critical(msg, exc_info=err)

    def get_context(self) -> Dict[str, Any]:
        context = model_to_dict(self.job)

        for key, value in model_to_dict(self.job.project).items():
            context[f"project__{key}"] = value

        context["project__id"] = self.job.project.id

        return context

    def get_command(self) -> List[str]:
        return [
            p % self.get_context() for p in ["python3", "entrypoint.py", *self.command]
        ]

    def before_docker_run(self) -> None:
        pass

    def after_docker_run(self) -> None:
        pass

    def after_docker_exception(self) -> None:
        pass

    def run(self):
        feedback = {}

        try:
            self.job.status = Job.Status.STARTED
            self.job.started_at = timezone.now()
            self.job.save()

            self.before_docker_run()

            command = self.get_command()
            volumes = []
            volumes.append(f"{str(self.shared_tempdir)}:/io/:rw")

            exit_code, output = self._run_docker(
                command,
                volumes=volumes,
            )

            if exit_code == TIMEOUT_ERROR_EXIT_CODE:
                feedback["error"] = "Worker timeout error."
                feedback["error_origin"] = "container"
                feedback["error_stack"] = ""
            else:
                try:
                    with open(self.shared_tempdir.joinpath("feedback.json"), "r") as f:
                        feedback = json.load(f)

                        if feedback.get("error"):
                            feedback["error_origin"] = "container"
                except Exception as err:
                    if not isinstance(feedback, dict):
                        feedback = {"error_feedback": feedback}

                    (_type, _value, tb) = sys.exc_info()
                    feedback["error"] = str(err)
                    feedback["error_origin"] = "worker_wrapper"
                    feedback["error_stack"] = traceback.format_tb(tb)

            feedback["container_exit_code"] = exit_code

            self.job.output = output.decode("utf-8")
            self.job.feedback = feedback
            self.job.finished_at = timezone.now()

            if exit_code != 0 or feedback.get("error") is not None:
                self.job.status = Job.Status.FAILED

                try:
                    self.after_docker_exception()
                except Exception as err:
                    logger.error(
                        "Failed to run the `after_docker_exception` handler.",
                        exc_info=err,
                    )

                self.job.save()
                return

            self.job.status = Job.Status.FINISHED
            self.job.save()

            self.after_docker_run()

        except Exception as err:
            (_type, _value, tb) = sys.exc_info()
            feedback["error"] = str(err)
            feedback["error_origin"] = "worker_wrapper"
            feedback["error_stack"] = traceback.format_tb(tb)

            if isinstance(err, requests.exceptions.ReadTimeout):
                feedback["error_timeout"] = True

            logger.error(
                f"Failed job run:\n{json.dumps(feedback, sort_keys=True)}", exc_info=err
            )

            try:
                self.job.status = Job.Status.FAILED
                self.job.feedback = feedback
                self.job.finished_at = timezone.now()

                try:
                    self.after_docker_exception()
                except Exception as err:
                    logger.error(
                        "Failed to run the `after_docker_exception` handler.",
                        exc_info=err,
                    )

                self.job.save()
            except Exception as err:
                logger.error(
                    "Failed to handle exception and update the job status", exc_info=err
                )

    def _run_docker(
        self, command: List[str], volumes: List[str], run_opts: Dict[str, Any] = {}
    ) -> Tuple[int, bytes]:
        QGIS_CONTAINER_NAME = os.environ.get("QGIS_CONTAINER_NAME", None)
        QFIELDCLOUD_HOST = os.environ.get("QFIELDCLOUD_HOST", None)
        QFIELDCLOUD_WORKER_QFIELDCLOUD_URL = os.environ.get(
            "QFIELDCLOUD_WORKER_QFIELDCLOUD_URL", None
        )
        TRANSFORMATION_GRIDS_VOLUME_NAME = os.environ.get(
            "TRANSFORMATION_GRIDS_VOLUME_NAME", None
        )

        assert QGIS_CONTAINER_NAME
        assert QFIELDCLOUD_HOST
        assert QFIELDCLOUD_WORKER_QFIELDCLOUD_URL
        assert TRANSFORMATION_GRIDS_VOLUME_NAME

        token = AuthToken.objects.create(
            user=self.job.created_by,
            client_type=AuthToken.ClientType.WORKER,
            expires_at=timezone.now() + timedelta(seconds=self.container_timeout_secs),
        )

        client = docker.from_env()

        extra_envvars = {}
        pgservice_file_contents = ""
        for secret in self.job.project.secrets.all():
            if secret.type == Secret.Type.ENVVAR:
                extra_envvars[secret.name] = secret.value
            elif secret.type == Secret.Type.PGSERVICE:
                pgservice_file_contents += f"\n{secret.value}"
            else:
                raise NotImplementedError(f"Unknown secret type: {secret.type}")

        logger.info(f"Execute: {' '.join(command)}")
        volumes.append(f"{TRANSFORMATION_GRIDS_VOLUME_NAME}:/transformation_grids:ro")

        container: Container = client.containers.run(  # type:ignore
            QGIS_CONTAINER_NAME,
            command,
            environment={
                "PGSERVICE_FILE_CONTENTS": pgservice_file_contents,
                "QFIELDCLOUD_TOKEN": token.key,
                "QFIELDCLOUD_URL": QFIELDCLOUD_WORKER_QFIELDCLOUD_URL,
                "JOB_ID": self.job_id,
                "PROJ_DOWNLOAD_DIR": "/transformation_grids",
                "QT_QPA_PLATFORM": "offscreen",
            },
            volumes=volumes,
            # TODO keep the logs somewhere or even better -> pipe them to redis and store them there
            # auto_remove=True,
            network=os.environ.get("QFIELDCLOUD_DEFAULT_NETWORK"),
            detach=True,
        )

        logger.info(f"Starting worker {container.id} ...")

        response = {"StatusCode": TIMEOUT_ERROR_EXIT_CODE}

        try:
            # will throw an ConnectionError, but the container is still alive
            response = container.wait(timeout=self.container_timeout_secs)
        except Exception as err:
            logger.exception("Timeout error.", exc_info=err)

        logs = container.logs()
        container.stop()
        container.remove()
        logger.info(
            f"Finished execution with code {response['StatusCode']}, logs:\n{logs}"
        )

        if response["StatusCode"] == TIMEOUT_ERROR_EXIT_CODE:
            logs += f"\nTimeout error! The job failed to finish within {self.container_timeout_secs} seconds!\n".encode()

        return response["StatusCode"], logs


class PackageJobRun(JobRun):
    job_class = PackageJob
    command = ["package", "%(project__id)s", "%(project__project_filename)s"]
    data_last_packaged_at = None

    def before_docker_run(self) -> None:
        # at the start of docker we assume we make the snapshot of the data
        self.data_last_packaged_at = timezone.now()

    def after_docker_run(self) -> None:
        # only successfully finished packaging jobs should update the Project.data_last_packaged_at
        if self.job.status == Job.Status.FINISHED:
            self.job.project.data_last_packaged_at = self.data_last_packaged_at
            self.job.project.last_package_job = self.job
            self.job.project.save()

            try:
                project_id = str(self.job.project.id)
                package_ids = storage.get_stored_package_ids(project_id)
                job_ids = [
                    str(job["id"])
                    for job in Job.objects.filter(
                        type=Job.Type.PACKAGE,
                    )
                    .exclude(
                        status__in=(Job.Status.FAILED, Job.Status.FINISHED),
                    )
                    .values("id")
                ]

                for package_id in package_ids:
                    # keep the last package
                    if package_id == str(self.job.project.last_package_job_id):
                        continue

                    # the job is still active, so it might be one of the new packages
                    if package_id in job_ids:
                        continue

                    storage.delete_stored_package(project_id, package_id)
            except Exception as err:
                logger.error(
                    "Failed to delete dangling packages, will be deleted via CRON later.",
                    exc_info=err,
                )


class DeltaApplyJobRun(JobRun):
    job_class = ApplyJob
    command = ["delta_apply", "%(project__id)s", "%(project__project_filename)s"]

    def __init__(self, job_id: str) -> None:
        super().__init__(job_id)

        if self.job.overwrite_conflicts:
            self.command = [*self.command, "--overwrite-conflicts"]

    def _prepare_deltas(self, deltas: Iterable[Delta]):
        delta_contents = []
        delta_client_ids = []

        for delta in deltas:
            delta_contents.append(delta.content)

            if "clientId" in delta.content:
                delta_client_ids.append(delta.content["clientId"])

        local_to_remote_pk_deltas = Delta.objects.filter(
            content__clientId__in=delta_client_ids,
            last_modified_pk__isnull=False,
        ).values("content__clientId", "content__localPk", "last_modified_pk")

        client_pks_map = {}

        for delta in local_to_remote_pk_deltas:
            key = f"{delta['content__clientId']}__{delta['content__localPk']}"
            client_pks_map[key] = delta["last_modified_pk"]

        deltafile_contents = {
            "deltas": delta_contents,
            "files": [],
            "id": str(uuid.uuid4()),
            "project": str(self.job.project.id),
            "version": "1.0",
            "clientPks": client_pks_map,
        }

        return deltafile_contents

    @transaction.atomic()
    def before_docker_run(self) -> None:
        deltas = self.job.deltas_to_apply.all()
        deltafile_contents = self._prepare_deltas(deltas)

        self.delta_ids = [d.id for d in deltas]

        ApplyJobDelta.objects.filter(
            apply_job_id=self.job_id,
            delta_id__in=self.delta_ids,
        ).update(status=Delta.Status.STARTED)

        self.job.deltas_to_apply.update(last_status=Delta.Status.STARTED)

        with open(self.shared_tempdir.joinpath("deltafile.json"), "w") as f:
            json.dump(deltafile_contents, f)

    def after_docker_run(self) -> None:
        delta_feedback = self.job.feedback["outputs"]["apply_deltas"]["delta_feedback"]
        is_data_modified = False

        for feedback in delta_feedback:
            delta_id = feedback["delta_id"]
            status = feedback["status"]
            modified_pk = feedback["modified_pk"]

            if status == "status_applied":
                status = Delta.Status.APPLIED
                is_data_modified = True
            elif status == "status_conflict":
                status = Delta.Status.CONFLICT
            elif status == "status_apply_failed":
                status = Delta.Status.NOT_APPLIED
            else:
                status = Delta.Status.ERROR
                # not certain what happened
                is_data_modified = True

            Delta.objects.filter(pk=delta_id).update(
                last_status=status,
                last_feedback=feedback,
                last_modified_pk=modified_pk,
                last_apply_attempt_at=self.job.started_at,
                last_apply_attempt_by=self.job.created_by,
            )

            ApplyJobDelta.objects.filter(
                apply_job_id=self.job_id,
                delta_id=delta_id,
            ).update(
                status=status,
                feedback=feedback,
                modified_pk=modified_pk,
            )

        if is_data_modified:
            self.job.project.data_last_updated_at = timezone.now()
            self.job.project.save()

    def after_docker_exception(self) -> None:
        Delta.objects.filter(
            id__in=self.delta_ids,
        ).update(last_status=Delta.Status.ERROR)

        ApplyJobDelta.objects.filter(
            apply_job_id=self.job_id,
            delta_id__in=self.delta_ids,
        ).update(
            status=Delta.Status.ERROR,
        )


class ProcessProjectfileJobRun(JobRun):
    job_class = ProcessProjectfileJob
    command = [
        "process_projectfile",
        "%(project__id)s",
        "%(project__project_filename)s",
    ]

    def get_context(self, *args) -> Dict[str, Any]:
        context = super().get_context(*args)

        if not context.get("project__project_filename"):
            context["project__project_filename"] = get_qgis_project_file(
                context["project__id"]
            )

        return context

    def after_docker_run(self) -> None:
        project = self.job.project
        project.project_details = self.job.feedback["outputs"]["project_details"][
            "project_details"
        ]

        thumbnail_filename = self.shared_tempdir.joinpath("thumbnail.png")
        with open(thumbnail_filename, "rb") as f:
            thumbnail_uri = qfieldcloud.core.utils2.storage.upload_project_thumbail(
                project, f, "image/png", "thumbnail"
            )
        project.thumbnail_uri = project.thumbnail_uri or thumbnail_uri
        project.save()

    def after_docker_exception(self) -> None:
        project = self.job.project
        project.project_details = None
        project.save()


# docker-app/qfieldcloud/core/management/commands/calcprojectstorage.py

import uuid

from django.core.management.base import BaseCommand
from qfieldcloud.core.models import Project


class Command(BaseCommand):
    """
    Recalculate projects storage size
    """

    def add_arguments(self, parser):
        parser.add_argument("project_id", type=uuid.UUID, nargs="?")
        parser.add_argument("--force-recalculate", action="store_true")

    def handle(self, *args, **options):
        project_id = options.get("project_id")
        force_recalculate = options.get("force_recalculate")

        extra_filters = {}
        if project_id:
            extra_filters["id"] = project_id

        if not project_id and not force_recalculate:
            extra_filters["storage_size_mb"] = 0

        for project in Project.objects.filter(
            project_filename__isnull=False,
            **extra_filters,
        ).order_by("-updated_at"):
            print(f'Calculating project files storage size for "{project.id}"...')
            project.save(recompute_storage=True)
            print(
                f'Project files storage size for "{project.id}" is {project.storage_size_mb}MB'
            )


# docker-app/qfieldcloud/core/management/commands/createuser.py

from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand


class Command(BaseCommand):
    """
    Creates a normal or super user using the CLI.
    Unlike the Django's createsuperuser command, here we can pass the password as an argument.
    This is a utility function that is expected to be used only for testing purposes.
    """

    help = """
        Create a user with given username, email and password
        Usage: python manage.py createuser --username=test --email=test@test.com --password=test --superuser
    """

    def add_arguments(self, parser):
        parser.add_argument("--username", type=str, required=True)
        parser.add_argument("--password", type=str, required=True)
        parser.add_argument("--email", type=str, required=True)
        parser.add_argument("--superuser", action="store_true")

    def handle(self, *args, **options):
        username = options.get("username")
        password = options.get("password")
        email = options.get("email")
        is_superuser = options.get("superuser")
        try:
            User = get_user_model()
            if not User.objects.filter(username=username).exists():
                User.objects.create_user(
                    username=username,
                    email=email,
                    password=password,
                    is_superuser=is_superuser,
                )
                print(f"User {username} has been successfully created\n")
            else:
                print(f"User {username} already exists\n")
        except Exception as e:
            print("ERROR: Unable to create user\n%s\n" % e)
            exit(1)


# docker-app/qfieldcloud/core/management/commands/createuseraccounts.py

from django.core.management.base import BaseCommand
from qfieldcloud.core.models import User, UserAccount
from qfieldcloud.subscription.models import Plan


class Command(BaseCommand):
    """
    Creates user accounts for all users that are missing a user account.
    """

    def handle(self, *args, **options):
        for user in User.objects.filter(
            useraccount=None, type__in=[User.TYPE_USER, User.TYPE_ORGANIZATION]
        ):
            print(
                f'Creating user account for user "{user.username}" email "{user.email}"...'
            )
            plan = Plan.objects.get(user_type=user.user_type, is_default=True)

            UserAccount.objects.create(user=user, plan=plan)


# docker-app/qfieldcloud/core/management/commands/dequeue.py

import logging
import signal
from time import sleep

from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand
from django.db import transaction
from django.db.models import Count, Q
from qfieldcloud.core.models import Job
from worker_wrapper.wrapper import (
    DeltaApplyJobRun,
    PackageJobRun,
    ProcessProjectfileJobRun,
)

SECONDS = 5


class GracefulKiller:
    alive = True

    def __init__(self):
        signal.signal(signal.SIGINT, self._kill)
        signal.signal(signal.SIGTERM, self._kill)

    def _kill(self, *args):
        self.alive = False


class Command(BaseCommand):
    help = "Dequeue QFieldCloud Jobs from the DB"

    def add_arguments(self, parser):
        parser.add_argument(
            "--single-shot", action="store_true", help="Don't run infinite loop."
        )

    def handle(self, *args, **options):
        logging.info("Dequeue QFieldCloud Jobs from the DB")
        killer = GracefulKiller()

        while killer.alive:
            # the worker-wrapper caches outdated ContentType ids during tests since
            # the worker-wrapper and the tests reside in different containers
            if settings.DATABASES["default"]["NAME"].startswith("test_"):
                ContentType.objects.clear_cache()

            queued_job = None

            with transaction.atomic():

                busy_projects_ids_qs = (
                    Job.objects.filter(
                        status=Job.Status.PENDING,
                    )
                    .annotate(
                        active_jobs_count=Count(
                            "project__jobs",
                            filter=Q(
                                project__jobs__status__in=[
                                    Job.Status.QUEUED,
                                    Job.Status.STARTED,
                                ]
                            ),
                        )
                    )
                    .filter(active_jobs_count__gt=0)
                    .values("active_jobs_count", "project_id")
                )

                busy_project_ids = [j["project_id"] for j in busy_projects_ids_qs]

                # select all the pending jobs, that their project has no other active job
                jobs_qs = (
                    Job.objects.select_for_update(skip_locked=True)
                    .filter(status=Job.Status.PENDING)
                    .exclude(project_id__in=busy_project_ids)
                )

                for job in jobs_qs:
                    queued_job = job

                    logging.info(f"Dequeued job {job.id}, run!")

                    job.status = Job.Status.QUEUED
                    job.save()
                    break

            if queued_job:
                self._run(queued_job)
                queued_job = None
            else:
                if options["single_shot"]:
                    break

                for _i in range(SECONDS):
                    if killer.alive:
                        sleep(1)

            if options["single_shot"]:
                break

    def run(self, job_id, *args, **options):
        try:
            job = Job.objects.get(id=job_id)
            self._run(job)
        except Exception:
            pass

    def _run(self, job: Job):
        job_run_classes = {
            Job.Type.PACKAGE: PackageJobRun,
            Job.Type.DELTA_APPLY: DeltaApplyJobRun,
            Job.Type.PROCESS_PROJECTFILE: ProcessProjectfileJobRun,
        }

        if job.type in job_run_classes:
            job_run_class = job_run_classes[job.type]
        else:
            raise NotImplementedError(f"Unknown job type {job.type}")

        job_run = job_run_class(job.id)
        job_run.run()


# docker-app/qfieldcloud/core/management/commands/inviteusers.py

from datetime import datetime

from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand
from qfieldcloud.core.invitations_utils import invite_user_by_email


class Command(BaseCommand):
    """
    Invite one or more users using the CLI.
    """

    help = """
        Create a user with given username, email and password
        Usage: python manage.py createuser --username=user1 --emails=test1@test.com test2@test.com --exit-on-failure
    """

    def add_arguments(self, parser):
        parser.add_argument("--inviter", type=str, required=True)
        parser.add_argument("--limit", type=int, default=30)
        parser.add_argument("--emails", type=str, nargs="+", required=True)
        parser.add_argument("--exit-on-failure", action="store_true")

    def handle(self, *args, **options):
        User = get_user_model()

        inviter_username = options.get("inviter")
        emails = options.get("emails", [])
        exit_on_failure = options.get("exit-on-failure")
        sent_emails_limit = options.get("limit", 0)

        try:
            inviter = User.objects.get(username=inviter_username)
        except User.DoesNotExist:
            print(f'ERROR: Failed to find user "{inviter_username}"!')
            exit(1)

        sent_emails_count = 0

        for email in emails:
            if sent_emails_count >= sent_emails_limit:
                break

            success, message = invite_user_by_email(email, inviter)

            if success:
                sent_emails_count += 1
                print(
                    f"{datetime.now().isoformat()}\tSUCCESS\tinvitation sent to {email}."
                )
            else:
                print(
                    f"{datetime.now().isoformat()}\tWARNING\tinvitation not sent to {email}. {message}"
                )

                if exit_on_failure:
                    exit(1)


# docker-app/qfieldcloud/core/management/commands/purge_old_file_versions.py

from django.core.management.base import BaseCommand, CommandError
from qfieldcloud.core.models import Project
from qfieldcloud.core.utils2 import storage


class Command(BaseCommand):
    """Runs purge_old_file_versions as a management command"""

    help = storage.purge_old_file_versions.__doc__

    PROMPT_TXT = "This will purge old files for all projects. Rerun with --force, or type 'yes' to continue, or 'no' to cancel: "

    def add_arguments(self, parser):
        parser.add_argument(
            "--projects",
            type=str,
            help="Comma separated list of ids of projects to prune. If unset, will purge all projects",
        )
        parser.add_argument(
            "--force",
            action="store_true",
            help="Prevent confirmation prompt when purging all projects",
        )

    def handle(self, *args, **options):

        # Determine project ids to work on
        proj_ids = options.get("projects")

        # Get the affected projects
        if not proj_ids:
            if options.get("force") is not True and input(Command.PROMPT_TXT) != "yes":
                raise CommandError("Collecting static files cancelled.")
            proj_instances = Project.objects.all()
        else:
            proj_instances = Project.objects.filter(pk__in=proj_ids.split(","))

        # Iterate through projects
        proj_instances = proj_instances.prefetch_related("owner__useraccount")
        for proj_instance in proj_instances:
            storage.purge_old_file_versions(proj_instance)

        print("done !")


# docker-app/qfieldcloud/core/management/commands/status.py

from django.conf import settings
from django.core.management.base import BaseCommand
from qfieldcloud.core import geodb_utils, utils


class Command(BaseCommand):
    help = "Check qfieldcloud status"

    def handle(self, *args, **options):
        results = {}

        results["redis"] = "ok"
        # Check if redis is visible
        if not utils.redis_is_running():
            results["redis"] = "error"

        results["geodb"] = "ok"
        # Check geodb
        if not geodb_utils.geodb_is_running():
            results["geodb"] = "error"

        results["storage"] = "ok"
        # Check if bucket exists (i.e. the connection works)
        try:
            s3_client = utils.get_s3_client()
            s3_client.head_bucket(Bucket=settings.STORAGE_BUCKET_NAME)
        except Exception:
            results["storage"] = "error"

        self.stdout.write(
            self.style.SUCCESS("Everything seems to work properly: {}".format(results))
        )


# docker-app/qfieldcloud/authentication/apps.py

from django.apps import AppConfig


class AuthConfig(AppConfig):
    name = "qfieldcloud.authentication"
    initialized = False

    @classmethod
    def initialize(cls):
        """
        Initialize authentication.
        This method is re-entrant and can be called multiple times.
        """

        if cls.initialized:
            return
        cls.initialized = True

        from .conf import settings  # noqa

    def ready(self):
        self.initialize()


# docker-app/qfieldcloud/authentication/authentication.py

from typing import Type

from django.contrib.auth import get_user_model
from django.http.request import HttpRequest
from django.utils import timezone
from django.utils.translation import gettext as _
from rest_framework import exceptions
from rest_framework.authentication import (
    TokenAuthentication as DjangoRestFrameworkTokenAuthentication,
)

from .models import AuthToken

User = get_user_model()


def invalidate_all_tokens(user: User) -> int:
    now = timezone.now()
    return AuthToken.objects.filter(user=user, expires_at__gt=now).update(
        expires_at=now
    )


def create_token(
    token_model: Type[AuthToken],
    user: User,
    _serializer=None,
    request: HttpRequest = None,
) -> AuthToken:
    user_agent = ""
    client_type = AuthToken.ClientType.UNKNOWN

    if request:
        user_agent = request.META.get("HTTP_USER_AGENT", "")
        client_type = AuthToken.guess_client_type(user_agent)

    token = token_model.objects.create(
        user=user, client_type=client_type, user_agent=user_agent
    )

    return token


class TokenAuthentication(DjangoRestFrameworkTokenAuthentication):
    """
    Multi token authentication based on simple token based authentication.
    Clients should authenticate by passing the token key in the "Authorization"
    HTTP header, prepended with the string "Token ".  For example:
        Authorization: Token 401f7ac837da42b97f613d789819ff93537bee6a
    """

    model = AuthToken

    def authenticate_credentials(self, key):
        model = self.get_model()
        try:
            token = model.objects.select_related("user").get(key=key)
        except model.DoesNotExist:
            raise exceptions.AuthenticationFailed(_("Invalid token."))

        if not token.is_active:
            raise exceptions.AuthenticationFailed(_("Token has expired."))

        if not token.user.is_active:
            raise exceptions.AuthenticationFailed(_("User inactive or deleted."))

        # update the token last used time
        # NOTE the UPDATE may be performed already on the `token = model.objects.get(key=key)`, but we lose "token has expired" exception.
        token.last_used_at = timezone.now()
        token.save()

        return (token.user, token)


# docker-app/qfieldcloud/authentication/conf.py

from django.conf import settings

settings.QFIELDCLOUD_LOGIN_SERIALIZER = getattr(
    settings,
    "QFIELDCLOUD_LOGIN_SERIALIZER",
    "qfieldcloud.authentication.serializers.LoginSerializer",
)

settings.QFIELDCLOUD_TOKEN_SERIALIZER = getattr(
    settings,
    "QFIELDCLOUD_TOKEN_SERIALIZER",
    "qfieldcloud.authentication.serializers.TokenSerializer",
)

settings.QFIELDCLOUD_USER_SERIALIZER = getattr(
    settings,
    "QFIELDCLOUD_USER_SERIALIZER",
    "qfieldcloud.authentication.serializers.UserSerializer",
)


# docker-app/qfieldcloud/authentication/models.py

import re
from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.auth import get_user_model
from django.db import models
from django.utils import timezone
from django.utils.crypto import get_random_string
from django.utils.translation import gettext as _


def generate_token_key() -> str:
    return get_random_string(settings.AUTH_TOKEN_LENGTH)


def generate_token_expires_at() -> datetime:
    return timezone.now() + timedelta(hours=settings.AUTH_TOKEN_EXPIRATION_HOURS)


class AuthToken(models.Model):
    class ClientType(models.TextChoices):
        BROWSER = "browser", _("Browser")
        CLI = "cli", _("Command line interface")
        SDK = "sdk", _("SDK")
        QFIELD = "qfield", _("QField")
        QFIELDSYNC = "qfieldsync", _("QFieldSync")
        WORKER = "worker", _("Worker")
        UNKNOWN = "unknown", _("Unknown")

    @staticmethod
    def guess_client_type(user_agent: str) -> ClientType:
        if not user_agent:
            return AuthToken.ClientType.UNKNOWN

        if user_agent.startswith("qfield|"):
            return AuthToken.ClientType.QFIELD

        if user_agent.startswith("sdk|"):
            return AuthToken.ClientType.SDK

        if user_agent.startswith("cli|"):
            return AuthToken.ClientType.CLI

        if re.search(r" QGIS/[34]\d{4}(\/.*)?$", user_agent):
            return AuthToken.ClientType.QFIELDSYNC

        if re.search(
            r"Mozilla\/5.0 .+(AppleWebKit\/\d+.\d+ \(KHTML, like Gecko\)|Firefox\/[\d\.]+)",
            user_agent,
        ):
            return AuthToken.ClientType.BROWSER

        return AuthToken.ClientType.UNKNOWN

    single_token_clients = [
        ClientType.QFIELD,
        ClientType.QFIELDSYNC,
        ClientType.UNKNOWN,
    ]

    user = models.ForeignKey(
        get_user_model(),
        verbose_name=_("User"),
        related_name="auth_tokens",
        on_delete=models.CASCADE,
    )
    key = models.CharField(
        _("Token"),
        max_length=300,
        db_index=True,
        unique=True,
        default=generate_token_key,
    )
    created_at = models.DateTimeField(_("Created at"), auto_now_add=True)
    expires_at = models.DateTimeField(
        _("Expires at"), default=generate_token_expires_at
    )
    updated_at = models.DateTimeField(_("Updated at"), auto_now=True)
    last_used_at = models.DateTimeField(_("Last used at"), null=True)
    user_agent = models.TextField(_("User-Agent"), blank=True)
    client_type = models.CharField(
        max_length=32, choices=ClientType.choices, default=ClientType.UNKNOWN
    )

    @property
    def is_active(self) -> bool:
        return self.expires_at > timezone.now()

    class Meta:
        verbose_name = _("Token")
        verbose_name_plural = _("Tokens")
        ordering = ("-created_at",)

    def __str__(self):
        return self.key

    def save(self, *args, **kwargs) -> None:
        if self.client_type in self.single_token_clients:
            # expire all other tokens
            now = timezone.now()
            AuthToken.objects.filter(
                user=self.user,
                client_type=self.client_type,
                expires_at__gt=now,
            ).exclude(pk=self.pk).update(expires_at=now)
        return super().save(*args, **kwargs)


# docker-app/qfieldcloud/authentication/serializers.py

from django.conf import settings
from django.contrib.auth import authenticate, get_user_model
from django.utils.translation import gettext as _
from rest_framework import serializers
from rest_framework.exceptions import ValidationError

from .models import AuthToken

User = get_user_model()


class LoginSerializer(serializers.Serializer):
    """Based on https://github.com/Tivix/django-rest-auth/blob/rest_auth/serializers.py#L19"""

    username = serializers.CharField(required=False, allow_blank=True)
    email = serializers.EmailField(required=False, allow_blank=True)
    password = serializers.CharField(style={"input_type": "password"})

    def authenticate(self, **kwargs):
        return authenticate(self.context["request"], **kwargs)

    def _validate_email(self, email, password):
        user = None

        if email and password:
            user = self.authenticate(email=email, password=password)
        else:
            msg = _('Must include "email" and "password".')
            raise ValidationError(msg)

        return user

    def _validate_username(self, username, password):
        user = None

        if username and password:
            user = self.authenticate(username=username, password=password)
        else:
            msg = _('Must include "username" and "password".')
            raise ValidationError(msg)

        return user

    def _validate_username_email(self, username, email, password):
        user = None

        if email and password:
            user = self.authenticate(email=email, password=password)
        elif username and password:
            user = self.authenticate(username=username, password=password)
        else:
            msg = _('Must include either "username" or "email" and "password".')
            raise ValidationError(msg)

        return user

    def validate(self, attrs):
        username = attrs.get("username")
        email = attrs.get("email")
        password = attrs.get("password")

        user = None

        if "allauth" in settings.INSTALLED_APPS:
            from allauth.account import app_settings

            # Authentication through email
            if (
                app_settings.AUTHENTICATION_METHOD
                == app_settings.AuthenticationMethod.EMAIL
            ):
                user = self._validate_email(email, password)

            # Authentication through username
            elif (
                app_settings.AUTHENTICATION_METHOD
                == app_settings.AuthenticationMethod.USERNAME
            ):
                user = self._validate_username(username, password)

            # Authentication through either username or email
            else:
                user = self._validate_username_email(username, email, password)

        else:
            # Authentication without using allauth
            if email:
                try:
                    username = User.objects.get(email__iexact=email).get_username()
                except User.DoesNotExist:
                    pass

            if username:
                user = self._validate_username_email(username, "", password)

        # Did we get back an active user?
        if user:
            if not user.is_active:
                msg = _("User account is disabled.")
                raise ValidationError(msg)
        else:
            msg = _("Unable to log in with provided credentials.")
            raise ValidationError(msg)

        attrs["user"] = user
        return attrs


class TokenSerializer(serializers.ModelSerializer):
    expires_at = serializers.DateTimeField()
    token = serializers.CharField(source="key")

    class Meta:
        model = AuthToken
        fields = (
            "token",
            "expires_at",
        )
        read_only_fields = (
            "token",
            "expires_at",
        )


class UserSerializer(serializers.ModelSerializer):
    """
    User model w/o password
    """

    class Meta:
        model = User
        fields = ("pk", "username", "email", "first_name", "last_name")
        read_only_fields = ("email",)


# docker-app/qfieldcloud/authentication/utils.py

from django.utils.module_loading import import_string


def load_module(name):
    try:
        backend = import_string(name)
        return backend
    except ModuleNotFoundError as e:
        raise ModuleNotFoundError(f'Can not find module path defined "{name}"') from e
    except ImportError as e:
        raise ImportError(f'Can not import backend class defined in "{name}"') from e


# docker-app/qfieldcloud/authentication/views.py

from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.exceptions import ObjectDoesNotExist
from django.utils import timezone
from django.utils.decorators import method_decorator
from django.utils.translation import gettext as _
from django.views.decorators.debug import sensitive_post_parameters
from rest_framework import status
from rest_framework.authtoken.views import ObtainAuthToken
from rest_framework.generics import RetrieveAPIView
from rest_framework.permissions import AllowAny, IsAuthenticated
from rest_framework.response import Response
from rest_framework.views import APIView

from .authentication import create_token
from .models import AuthToken
from .utils import load_module

LoginSerializer = load_module(settings.QFIELDCLOUD_LOGIN_SERIALIZER)
TokenSerializer = load_module(settings.QFIELDCLOUD_TOKEN_SERIALIZER)
UserSerializer = load_module(settings.QFIELDCLOUD_USER_SERIALIZER)

sensitive_post_parameters_m = method_decorator(
    sensitive_post_parameters(
        "password", "old_password", "new_password1", "new_password2"
    )
)


class LoginView(ObtainAuthToken):
    """Create a new user session.

    Check the credentials and return the REST Token if the credentials are valid and authenticated.
    Accept the following POST parameters: username OR email, password
    Return information about the token and the user.
    """

    # Based on: https://github.com/Tivix/django-rest-auth/blob/master/rest_auth/views.py#L33

    permission_classes = (AllowAny,)
    serializer_class = LoginSerializer
    token_model = AuthToken

    @sensitive_post_parameters_m
    def dispatch(self, *args, **kwargs):
        return super(LoginView, self).dispatch(*args, **kwargs)

    def post(self, request, *args, **kwargs):
        self.request = request
        self.serializer = self.get_serializer(
            data=self.request.data, context={"request": request}
        )
        self.serializer.is_valid(raise_exception=True)
        validated_data = self.serializer.validated_data
        assert validated_data and "user" in validated_data
        self.token = create_token(
            self.token_model, validated_data["user"], self.serializer, self.request
        )

        serializer = TokenSerializer(
            instance=self.token, context={"request": self.request}
        )
        return Response(serializer.data, status=status.HTTP_200_OK)


class LogoutView(APIView):
    """Invalidate the user session.

    Calls Django logout method and invalidate the Token object assigned to the current User object.
    Accepts nothing, returns a details message.
    """

    # Based on: https://github.com/Tivix/django-rest-auth/blob/master/rest_auth/views.py#L109

    permission_classes = (AllowAny,)

    def get(self, request, *args, **kwargs):
        if getattr(settings, "ACCOUNT_LOGOUT_ON_GET", False):
            response = self.logout(request)
        else:
            response = self.http_method_not_allowed(request, *args, **kwargs)

        return self.finalize_response(request, response, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        return self.logout(request)

    def logout(self, request):
        try:
            now = timezone.now()
            request.user.auth_tokens.filter(expires_at__gt=now).update(expires_at=now)
        except (AttributeError, ObjectDoesNotExist):
            pass

        response = Response(
            {"detail": _("Successfully logged out.")}, status=status.HTTP_200_OK
        )
        return response


class UserView(RetrieveAPIView):
    """Read user fields.

    Accepts nothing, returns the user fields.
    """

    # Based on: https://github.com/Tivix/django-rest-auth/blob/master/rest_auth/views.py#L146

    serializer_class = UserSerializer
    permission_classes = (IsAuthenticated,)

    def get_object(self):
        return self.request.user

    def get_queryset(self):
        """
        Adding this method since it is sometimes called when using
        django-rest-swagger
        https://github.com/Tivix/django-rest-auth/issues/275
        """
        return get_user_model().objects.none()


# docker-app/qfieldcloud/notifs/admin.py

from notifications.admin import NotificationAdmin

# Override django-notification's list display
NotificationAdmin.list_display = (
    "actor",
    "verb",
    "action_object",
    "target",
    "timestamp",
    "recipient",
    "unread",
)


# docker-app/qfieldcloud/notifs/apps.py

from django.apps import AppConfig


class NotifsConfig(AppConfig):
    name = "qfieldcloud.notifs"
    verbose_name = "Notifs"

    def ready(self):
        from . import signals  # noqa


# docker-app/qfieldcloud/notifs/cron.py

import logging

from django.conf import settings
from django.core.mail import send_mail
from django.db.models import Exists
from django.db.models.expressions import OuterRef
from django.db.models.functions import Now
from django.template.loader import render_to_string
from django_cron import CronJobBase, Schedule
from notifications.models import Notification
from qfieldcloud.core.models import User


class SendNotificationsJob(CronJobBase):
    schedule = Schedule(run_every_mins=1)
    code = "qfieldcloud.send_notifications"

    # TODO : not sure if/how this is logged somewhere
    def do(self):
        try:

            users = User.objects.filter(user_type=User.TYPE_USER).filter(
                Exists(
                    Notification.objects.filter(
                        unread=True,
                        emailed=False,
                        timestamp__lte=Now()
                        - OuterRef("useraccount__notifs_frequency"),
                    )
                )
            )
            for user in users:
                logging.debug(f"Retrieving notifications for {user}")

                notifs = Notification.objects.filter(
                    recipient=user, unread=True, emailed=False
                )
                if not notifs:
                    logging.debug(f"{user} has no notifications.")
                    continue

                if not user.email:
                    logging.warning(f"{user} has notifications, but no email set !")
                    continue

                logging.debug(f"Sending an email to {user} !")

                context = {
                    "notifs": notifs,
                    "username": user.username,
                    "hostname": settings.ALLOWED_HOSTS[0],
                }

                subject = render_to_string(
                    "notifs/notification_email_subject.txt", context
                )
                body_html = render_to_string(
                    "notifs/notification_email_body.html", context
                )
                body_plain = render_to_string(
                    "notifs/notification_email_body.txt", context
                )

                # TODO : use mass_send_mail
                send_mail(
                    subject.strip(),
                    body_plain,
                    settings.DEFAULT_FROM_EMAIL,
                    [user.email],
                    html_message=body_html,
                )

                notifs.update(emailed=True)

        except Exception as e:
            logging.exception(e)
            raise e


# docker-app/qfieldcloud/notifs/signals.py

"""
This module logs activity streams using the `django-activity-stream` package.
"""
from django.db.models.signals import post_save, pre_delete
from django.dispatch import receiver
from django_currentuser.middleware import get_current_authenticated_user
from notifications.signals import notify

from ..core.models import (
    Organization,
    OrganizationMember,
    Project,
    ProjectCollaborator,
    ProjectQueryset,
    Team,
    TeamMember,
    User,
)


def _send_notif(verb, action_object, recipient, target=None):
    """
    Sends a notification through django-notifications, but sets it as already read
    if the actor is the same as the recipient
    """
    authenticated_user = get_current_authenticated_user()

    # django-notifications-hq doesn't support Anonymous users yet (e.g. management command)
    if authenticated_user is None:
        return

    # deduplicate
    recipient = list(set(recipient))

    if authenticated_user in recipient:
        # If the actor is already the recipient, we consider the notification as seen

        # TODO : marking notifications as read is currently not supported, so we disable
        # self-notifications for now
        # see https://github.com/django-notifications/django-notifications/issues/317
        # notify.send(
        #     authenticated_user,
        #     verb=verb,
        #     action_object=action_object,
        #     target=target,
        #     recipient=[authenticated_user],
        #     unread=False, # not supported !
        # )
        recipient.remove(authenticated_user)

    notify.send(
        authenticated_user,
        verb=verb,
        action_object=action_object,
        target=target,
        recipient=list(recipient),
    )


def _concerned_users_in_entity(entity: User):
    """Returns a list of users (of TYPE_USER) concerned by updates to an user (any type)"""

    return User.objects.for_entity(entity).filter(user_type=User.TYPE_USER)


def _concerned_users_in_project(project: Project):
    """Returns a list of users concerned by updates to a project"""

    return (
        User.objects.for_project(project)
        .filter(user_type=User.TYPE_USER)
        .exclude(project_role_origin=ProjectQueryset.RoleOrigins.PUBLIC)
    )


# Register signals
@receiver(post_save, sender=User)
def user_created(sender, instance, created, **kwargs):
    user = instance
    _send_notif(
        verb="created",
        action_object=user,
        recipient=_concerned_users_in_entity(user),
    )


# Organizations


@receiver(post_save, sender=Organization)
def organization_created(sender, instance, created, **kwargs):
    organization = instance
    # We don't notify for simple updates
    if not created:
        return

    _send_notif(
        verb="created",
        action_object=organization,
        recipient=_concerned_users_in_entity(organization),
    )


@receiver(pre_delete, sender=Organization)
def organization_deleted(sender, instance, **kwargs):
    organization = instance

    _send_notif(
        verb="deleted",
        action_object=organization,
        recipient=_concerned_users_in_entity(organization),
    )


@receiver(post_save, sender=OrganizationMember)
def organization_membership_created(sender, instance, created, **kwargs):
    membership = instance
    # We don't notify for simple updates
    if not created:
        return

    _send_notif(
        verb="added",
        action_object=membership.member,
        target=membership.organization,
        recipient=_concerned_users_in_entity(membership.organization),
    )


@receiver(pre_delete, sender=OrganizationMember)
def organization_membership_deleted(sender, instance, **kwargs):
    membership = instance
    _send_notif(
        verb="removed",
        action_object=membership.member,
        target=membership.organization,
        recipient=_concerned_users_in_entity(membership.organization),
    )


# Teams


@receiver(post_save, sender=Team)
def team_created(sender, instance, created, **kwargs):
    team = instance
    # We don't notify for simple updates
    if not created:
        return

    _send_notif(
        verb="created",
        action_object=team,
        recipient=_concerned_users_in_entity(team),
    )


@receiver(pre_delete, sender=Team)
def team_deleted(sender, instance, **kwargs):
    team = instance
    _send_notif(
        verb="deleted",
        action_object=team,
        recipient=_concerned_users_in_entity(team),
    )


@receiver(post_save, sender=TeamMember)
def team_membership_created(sender, instance, created, **kwargs):
    membership = instance
    # We don't notify for simple updates
    if not created:
        return

    _send_notif(
        verb="added",
        action_object=membership.member,
        target=membership.team,
        recipient=_concerned_users_in_entity(membership.team),
    )


@receiver(pre_delete, sender=TeamMember)
def team_membership_deleted(sender, instance, **kwargs):
    membership = instance
    _send_notif(
        verb="removed",
        action_object=membership.member,
        target=membership.team,
        recipient=_concerned_users_in_entity(membership.team),
    )


# Projects


@receiver(post_save, sender=Project)
def project_created(sender, instance, created, **kwargs):
    project = instance
    # We don't notify for simple updates
    if not created:
        return

    _send_notif(
        verb="created",
        action_object=project,
        recipient=_concerned_users_in_project(project),
    )


@receiver(pre_delete, sender=Project)
def project_deleted(sender, instance, **kwargs):
    project = instance
    _send_notif(
        verb="deleted",
        action_object=project,
        recipient=_concerned_users_in_project(project),
    )


@receiver(post_save, sender=ProjectCollaborator)
def project_membership_created(sender, instance, created, **kwargs):
    membership = instance

    # We don't notify for simple updates
    if not created:
        return

    _send_notif(
        verb="added",
        action_object=membership.collaborator,
        target=membership.project,
        recipient=_concerned_users_in_project(membership.project),
    )


@receiver(pre_delete, sender=ProjectCollaborator)
def project_membership_deleted(sender, instance, **kwargs):
    membership = instance

    _send_notif(
        verb="removed",
        action_object=membership.collaborator,
        target=membership.project,
        recipient=_concerned_users_in_project(membership.project),
    )


# docker-app/qfieldcloud/subscription/admin.py

from django.contrib import admin
from django.contrib.admin import register

from .models import ExtraPackageTypeJobMinutes, ExtraPackageTypeStorage, Plan


@register(Plan)
class PlanAdmin(admin.ModelAdmin):
    list_display = [
        "code",
        "is_default",
        "is_public",
        "display_name",
        "storage_mb",
        "job_minutes",
    ]


@register(ExtraPackageTypeStorage, ExtraPackageTypeJobMinutes)
class ExtraPackageTypeAdmin(admin.ModelAdmin):
    list_display = [
        "code",
        "is_public",
        "display_name",
    ]


# docker-app/qfieldcloud/subscription/apps.py

from django.apps import AppConfig


class CoreConfig(AppConfig):
    name = "qfieldcloud.subscription"


# docker-app/qfieldcloud/subscription/models.py

from datetime import timedelta

from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models, transaction
from django.utils.translation import gettext as _


class Plan(models.Model):

    # NOTE the values for USER and ORGANIZATION should match the values of TYPE_USER and TYPE_ORGANIZATION in qfieldcloud.core.models.User
    class UserType(models.IntegerChoices):
        USER = (1, _("User"))
        ORGANIZATION = (2, _("Organization"))

    @classmethod
    def get_or_create_default(cls) -> "Plan":
        """Returns the default plan, creating one if none exists.
        To be used as a default value for UserAccount.type"""
        if cls.objects.count() == 0:
            with transaction.atomic():
                cls.objects.create(
                    code="default_user",
                    display_name="default user (autocreated)",
                    is_default=True,
                    is_public=False,
                    user_type=Plan.UserType.USER,
                )
                cls.objects.create(
                    code="default_org",
                    display_name="default organization (autocreated)",
                    is_default=True,
                    is_public=False,
                    user_type=Plan.UserType.ORGANIZATION,
                )
        return cls.objects.order_by("-is_default").first()

    # unique identifier of the subscription plan
    code = models.CharField(max_length=100, unique=True)

    # the plan would be applicable only to user of that `user_type`
    user_type = models.PositiveSmallIntegerField(
        choices=UserType.choices, default=UserType.USER
    )

    # relative ordering of the record
    ordering = models.PositiveIntegerField(
        default=0,
        help_text=_(
            'Relative ordering of the record. Lower values have higher priority (will be first in the list). Records with same ordering will be sorted by "Display name" and "Code". Please set with gaps for different records for easy reordering (e.g. 5, 10, 15, but not 5, 6, 7).'
        ),
    )

    # TODO: match requirements in QF-234 (fields like automatic old versions)
    # TODO: decide how to localize display_name. Possible approaches:
    # - django-vinaigrette (never tried, but like the name, and seems to to exactly what we want)
    # - django-modeltranslation (tried, works well, but maybe overkill as it creates new database columns for each locale)
    # - something else ? there's probably some json based stuff
    display_name = models.CharField(max_length=100)
    storage_mb = models.PositiveIntegerField(default=10)
    storage_keep_versions = models.PositiveIntegerField(default=10)
    job_minutes = models.PositiveIntegerField(default=10)
    can_add_storage = models.BooleanField(default=False)
    can_add_job_minutes = models.BooleanField(default=False)
    is_external_db_supported = models.BooleanField(default=False)
    can_configure_repackaging_cache_expire = models.BooleanField(default=False)
    min_repackaging_cache_expire = models.DurationField(
        default=timedelta(minutes=60),
        validators=[MinValueValidator(timedelta(minutes=1))],
    )
    synchronizations_per_months = models.PositiveIntegerField(default=30)

    # the plan is visible option for non-admin users
    is_public = models.BooleanField(default=False)

    # the plan is set by default for new users
    is_default = models.BooleanField(default=False)

    # the plan is marked as premium which assumes it has premium access
    is_premium = models.BooleanField(default=False)

    # The maximum number of organizations members that are allowed to be added per organization
    # This constraint is useful for public administrations with limited resources who want to cap
    # the maximum amount of money that they are going to pay.
    # Only makes sense when the user_type == UserType.ORGANIZATION
    # If the organization subscription is changed from unlimited to limited organization members,
    # the existing members that are over the max_organization_members configuration remain active.
    max_organization_members = models.IntegerField(
        default=-1,
        help_text=_(
            "Maximum organization members allowed. Set -1 to allow unlimited organization members."
        ),
    )

    # The maximum number of premium collaborators that are allowed to be added per project.
    # If the project owner's plan is changed from unlimited to limited organization members,
    # the existing members that are over the `max_premium_collaborators_per_private_project` configuration remain active.
    max_premium_collaborators_per_private_project = models.IntegerField(
        default=-1,
        help_text=_(
            "Maximum premium collaborators per private project. Set -1 to allow unlimited project collaborators."
        ),
    )

    def save(self, *args, **kwargs):
        with transaction.atomic():
            # If default is set to true, we unset default on all other plans
            if self.is_default:
                Plan.objects.filter(user_type=self.user_type).update(is_default=False)
            return super().save(*args, **kwargs)

    def __str__(self):
        return f"{self.display_name} ({self.code})"

    class Meta:
        ordering = (
            "ordering",
            "display_name",
            "code",
        )


class ExtraPackageType(models.Model):
    code = models.CharField(max_length=100, unique=True)
    # TODO: decide how to localize display_name. Possible approaches:
    # - django-vinaigrette (never tried, but like the name, and seems to to exactly what we want)
    # - django-modeltranslation (tried, works well, but maybe overkill as it creates new database columns for each locale)
    # - something else ? there's probably some json based stuff
    display_name = models.CharField(max_length=100)
    is_public = models.BooleanField(default=False)


class ExtraPackageTypeStorage(ExtraPackageType):
    megabytes = models.PositiveIntegerField()


class ExtraPackageTypeJobMinutes(ExtraPackageType):
    minutes = models.PositiveIntegerField()


class ExtraPackage(models.Model):
    account = models.ForeignKey(
        "core.UserAccount",
        on_delete=models.CASCADE,
        related_name="extra_packages",
    )
    type = models.ForeignKey(
        ExtraPackageType, on_delete=models.CASCADE, related_name="packages"
    )
    quantity = models.PositiveIntegerField(
        validators=[
            MinValueValidator(1),
            MaxValueValidator(25),
        ],
    )
    start_date = models.DateField()
    end_date = models.DateField(null=True, blank=True)


# docker-app/manage.py

#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys


def main():
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "qfieldcloud.settings")
    try:
        from django.core.management import execute_from_command_line
    except ImportError as exc:
        raise ImportError(
            "Couldn't import Django. Are you sure it's installed and "
            "available on your PYTHONPATH environment variable? Did you "
            "forget to activate a virtual environment?"
        ) from exc
    execute_from_command_line(sys.argv)


if __name__ == "__main__":
    main()


# docker-app/wait_for_services.py

import logging
import os
from time import sleep, time

import psycopg2
import redis

logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

TIMEOUT = 30
INTERVAL = 2


def wait_for_postgres():
    logger.info("Waiting for postgres...")
    config = {
        "dbname": os.environ.get("SQL_DATABASE"),
        "user": os.environ.get("SQL_USER"),
        "password": os.environ.get("SQL_PASSWORD"),
        "host": os.environ.get("SQL_HOST"),
        "port": os.environ.get("SQL_PORT"),
        "sslmode": os.environ.get("SQL_SSLMODE"),
    }
    start_time = time()
    while time() - start_time < TIMEOUT:
        try:
            conn = psycopg2.connect(**config)
            logger.info("Postgres is ready! ✨ 💅")
            conn.close()
            return True
        except psycopg2.OperationalError as e:
            logger.info(
                f"Postgres isn't ready.\n{e}\n Waiting for {INTERVAL} second(s)..."
            )
            sleep(INTERVAL)
    logger.error(f"We could not connect to Postgres within {TIMEOUT} seconds.")

    return False


wait_for_postgres()


def wait_for_redis():
    logger.info("Waiting for redis...")
    start_time = time()
    while time() - start_time < TIMEOUT:
        logger.info("Waiting for redis...")
        try:
            r = redis.Redis(
                host="redis", password=os.environ.get("REDIS_PASSWORD"), db=0
            )
            if not r.ping():
                raise Exception
            logger.info("Redis is ready! ✨ 💅")
            return True
        except Exception as e:
            logger.info("Redis isn't ready.\n%s" % e)
        sleep(INTERVAL)

    return False


wait_for_redis()


# docker-app/qfieldcloud/core/logging/formatters.py

import json_log_formatter
from django.core.handlers.wsgi import WSGIRequest
from django.core.serializers.json import DjangoJSONEncoder


class JsonEncoder(DjangoJSONEncoder):
    def default(self, obj):
        return super().default(obj)


class CustomisedJSONFormatter(json_log_formatter.JSONFormatter):
    def to_json(self, record):
        """Converts record dict to a JSON string.
        It makes best effort to serialize a record (represents an object as a string)
        instead of raising TypeError if json library supports default argument.
        Note, ujson doesn't support it.
        Override this method to change the way dict is converted to JSON.
        """
        try:
            return self.json_lib.dumps(
                record, default=json_default, sort_keys=True, cls=JsonEncoder
            )
        # ujson doesn't support default argument and raises TypeError.
        except TypeError:
            return self.json_lib.dumps(record, cls=JsonEncoder)


def json_default(obj):
    if isinstance(obj, WSGIRequest):
        return str(obj)

    try:
        return obj.__dict__
    except AttributeError:
        return str(obj)


# docker-app/qfieldcloud/core/middleware/test.py

import os

from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import MiddlewareNotUsed


class TestMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

        if os.environ.get("ENVIRONMENT") != "test":
            raise MiddlewareNotUsed()

    def __call__(self, request):
        ContentType.objects.clear_cache()
        return self.get_response(request)


# docker-app/qfieldcloud/core/middleware/timezone.py

import pytz
from django.conf import settings
from django.utils import timezone


class TimezoneMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        if request.user.is_authenticated and hasattr(
            request.user.useraccount, "useraccount"
        ):
            user_tz = request.user.useraccount.timezone
        elif settings.TIME_ZONE:
            user_tz = pytz.timezone(settings.TIME_ZONE)
        else:
            user_tz = None

        if user_tz:
            timezone.activate(user_tz)
        else:
            timezone.deactivate()

        return self.get_response(request)


# docker-app/qfieldcloud/core/migrations/0001_initial.py

# Generated by Django 2.2.15 on 2020-09-06 14:08

import uuid

import django.contrib.auth.models
import django.contrib.auth.validators
import django.core.validators
import django.db.models.deletion
import django.utils.timezone
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ("auth", "0011_update_proxy_permissions"),
    ]

    operations = [
        migrations.CreateModel(
            name="User",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("password", models.CharField(max_length=128, verbose_name="password")),
                (
                    "last_login",
                    models.DateTimeField(
                        blank=True, null=True, verbose_name="last login"
                    ),
                ),
                (
                    "is_superuser",
                    models.BooleanField(
                        default=False,
                        help_text="Designates that this user has all permissions without explicitly assigning them.",
                        verbose_name="superuser status",
                    ),
                ),
                (
                    "username",
                    models.CharField(
                        error_messages={
                            "unique": "A user with that username already exists."
                        },
                        help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.",
                        max_length=150,
                        unique=True,
                        validators=[
                            django.contrib.auth.validators.UnicodeUsernameValidator()
                        ],
                        verbose_name="username",
                    ),
                ),
                (
                    "first_name",
                    models.CharField(
                        blank=True, max_length=30, verbose_name="first name"
                    ),
                ),
                (
                    "last_name",
                    models.CharField(
                        blank=True, max_length=150, verbose_name="last name"
                    ),
                ),
                (
                    "email",
                    models.EmailField(
                        blank=True, max_length=254, verbose_name="email address"
                    ),
                ),
                (
                    "is_staff",
                    models.BooleanField(
                        default=False,
                        help_text="Designates whether the user can log into this admin site.",
                        verbose_name="staff status",
                    ),
                ),
                (
                    "is_active",
                    models.BooleanField(
                        default=True,
                        help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.",
                        verbose_name="active",
                    ),
                ),
                (
                    "date_joined",
                    models.DateTimeField(
                        default=django.utils.timezone.now, verbose_name="date joined"
                    ),
                ),
                (
                    "user_type",
                    models.PositiveSmallIntegerField(
                        choices=[(1, "user"), (2, "organization")], default=1
                    ),
                ),
                (
                    "groups",
                    models.ManyToManyField(
                        blank=True,
                        help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",
                        related_name="user_set",
                        related_query_name="user",
                        to="auth.Group",
                        verbose_name="groups",
                    ),
                ),
                (
                    "user_permissions",
                    models.ManyToManyField(
                        blank=True,
                        help_text="Specific permissions for this user.",
                        related_name="user_set",
                        related_query_name="user",
                        to="auth.Permission",
                        verbose_name="user permissions",
                    ),
                ),
            ],
            options={
                "verbose_name": "user",
                "verbose_name_plural": "users",
                "abstract": False,
            },
            managers=[
                ("objects", django.contrib.auth.models.UserManager()),
            ],
        ),
        migrations.CreateModel(
            name="Project",
            fields=[
                (
                    "id",
                    models.UUIDField(
                        default=uuid.uuid4,
                        editable=False,
                        primary_key=True,
                        serialize=False,
                    ),
                ),
                ("name", models.CharField(help_text="Project name", max_length=255)),
                ("description", models.TextField(blank=True)),
                ("private", models.BooleanField(default=False)),
                ("created_at", models.DateTimeField(auto_now_add=True)),
                ("updated_at", models.DateTimeField(auto_now=True)),
                (
                    "owner",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to=settings.AUTH_USER_MODEL,
                    ),
                ),
            ],
        ),
        migrations.CreateModel(
            name="Organization",
            fields=[
                (
                    "user_ptr",
                    models.OneToOneField(
                        auto_created=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        parent_link=True,
                        primary_key=True,
                        serialize=False,
                        to=settings.AUTH_USER_MODEL,
                    ),
                ),
                (
                    "organization_owner",
                    models.ForeignKey(
                        limit_choices_to=models.Q(user_type=1),
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="owner",
                        to=settings.AUTH_USER_MODEL,
                    ),
                ),
            ],
            options={
                "verbose_name": "organization",
                "verbose_name_plural": "organizations",
            },
            bases=("core.user",),
            managers=[
                ("objects", django.contrib.auth.models.UserManager()),
            ],
        ),
        migrations.CreateModel(
            name="ProjectCollaborator",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "role",
                    models.PositiveSmallIntegerField(
                        choices=[
                            (1, "admin"),
                            (2, "manager"),
                            (3, "editor"),
                            (4, "reporter"),
                            (5, "reader"),
                        ],
                        default=5,
                    ),
                ),
                (
                    "collaborator",
                    models.ForeignKey(
                        limit_choices_to=models.Q(user_type=1),
                        on_delete=django.db.models.deletion.CASCADE,
                        to=settings.AUTH_USER_MODEL,
                    ),
                ),
                (
                    "project",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="collaborators",
                        to="core.Project",
                    ),
                ),
            ],
        ),
        migrations.CreateModel(
            name="OrganizationMember",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "role",
                    models.PositiveSmallIntegerField(
                        choices=[(1, "admin"), (2, "member")], default=2
                    ),
                ),
                (
                    "member",
                    models.ForeignKey(
                        limit_choices_to=models.Q(user_type=1),
                        on_delete=django.db.models.deletion.CASCADE,
                        to=settings.AUTH_USER_MODEL,
                    ),
                ),
                (
                    "organization",
                    models.ForeignKey(
                        limit_choices_to=models.Q(user_type=2),
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="members",
                        to="core.Organization",
                    ),
                ),
            ],
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0002_deltafile.py

# Generated by Django 2.2.17 on 2020-11-16 16:01

import uuid

import django.contrib.postgres.fields.jsonb
import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0001_initial"),
    ]

    operations = [
        migrations.CreateModel(
            name="Deltafile",
            fields=[
                (
                    "id",
                    models.UUIDField(
                        default=uuid.uuid4,
                        editable=False,
                        primary_key=True,
                        serialize=False,
                    ),
                ),
                ("content", django.contrib.postgres.fields.jsonb.JSONField()),
                (
                    "status",
                    models.PositiveSmallIntegerField(
                        choices=[
                            (1, "PENDING"),
                            (2, "BUSY"),
                            (3, "APPLIED"),
                            (4, "APPLIED_WITH_CONFLICTS"),
                            (5, "NOT_APPLIED"),
                            (6, "ERROR"),
                        ],
                        default=1,
                    ),
                ),
                ("output", models.TextField(blank=True)),
                ("created_at", models.DateTimeField(auto_now_add=True)),
                ("updated_at", models.DateTimeField(auto_now=True)),
                (
                    "project",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="deltas",
                        to="core.Project",
                    ),
                ),
            ],
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0003_auto_20201127_1256.py

# Generated by Django 2.2.17 on 2020-11-27 12:56

import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0002_deltafile"),
    ]

    operations = [
        migrations.CreateModel(
            name="UserAccount",
            fields=[
                (
                    "user",
                    models.OneToOneField(
                        on_delete=django.db.models.deletion.CASCADE,
                        primary_key=True,
                        serialize=False,
                        to=settings.AUTH_USER_MODEL,
                    ),
                ),
                (
                    "account_type",
                    models.PositiveSmallIntegerField(
                        choices=[(1, "community"), (2, "pro")], default=1
                    ),
                ),
                ("storage_limit_mb", models.PositiveIntegerField(default=100)),
                ("db_limit_mb", models.PositiveIntegerField(default=25)),
                (
                    "synchronizations_per_months",
                    models.PositiveIntegerField(default=30),
                ),
            ],
        ),
        migrations.AlterField(
            model_name="deltafile",
            name="status",
            field=models.PositiveSmallIntegerField(
                choices=[
                    (1, "STATUS_PENDING"),
                    (2, "STATUS_BUSY"),
                    (3, "STATUS_APPLIED"),
                    (4, "STATUS_APPLIED_WITH_CONFLICTS"),
                    (5, "STATUS_NOT_APPLIED"),
                    (6, "STATUS_ERROR"),
                ],
                default=1,
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0004_auto_20201201_1209.py

# Generated by Django 2.2.17 on 2020-12-01 12:09

import uuid

import django.contrib.postgres.fields.jsonb
import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0003_auto_20201127_1256"),
    ]

    operations = [
        migrations.CreateModel(
            name="Delta",
            fields=[
                (
                    "id",
                    models.UUIDField(
                        default=uuid.uuid4,
                        editable=False,
                        primary_key=True,
                        serialize=False,
                    ),
                ),
                ("deltafile_id", models.UUIDField()),
                ("content", django.contrib.postgres.fields.jsonb.JSONField()),
                (
                    "status",
                    models.PositiveSmallIntegerField(
                        choices=[
                            (1, "STATUS_PENDING"),
                            (2, "STATUS_BUSY"),
                            (3, "STATUS_APPLIED"),
                            (4, "STATUS_APPLIED_WITH_CONFLICTS"),
                            (5, "STATUS_NOT_APPLIED"),
                            (6, "STATUS_ERROR"),
                        ],
                        default=1,
                    ),
                ),
                ("output", models.TextField(blank=True)),
                ("created_at", models.DateTimeField(auto_now_add=True)),
                ("updated_at", models.DateTimeField(auto_now=True)),
                (
                    "project",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="deltas",
                        to="core.Project",
                    ),
                ),
            ],
        ),
        migrations.DeleteModel(
            name="Deltafile",
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0005_auto_20201203_1037.py

# Generated by Django 2.2.17 on 2020-12-03 10:37

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0004_auto_20201201_1209"),
    ]

    operations = [
        migrations.AlterField(
            model_name="delta",
            name="status",
            field=models.PositiveSmallIntegerField(
                choices=[
                    (1, "STATUS_PENDING"),
                    (2, "STATUS_BUSY"),
                    (3, "STATUS_APPLIED"),
                    (4, "STATUS_CONFLICT"),
                    (5, "STATUS_NOT_APPLIED"),
                    (6, "STATUS_ERROR"),
                ],
                default=1,
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0006_auto_20201214_1642.py

# Generated by Django 2.2.17 on 2020-12-14 16:42

import json

import django.contrib.postgres.fields.jsonb
from django.db import migrations


def jsonify_output_field(apps, schema_editor):
    # Old values in output field of delta table where string instead of json
    Delta = apps.get_model("core", "Delta")
    for delta in Delta.objects.all():
        delta.output = json.dumps([{"msg": delta.output}])
        delta.save()


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0005_auto_20201203_1037"),
    ]

    operations = [
        migrations.RunPython(jsonify_output_field),
        migrations.AlterField(
            model_name="delta",
            name="output",
            field=django.contrib.postgres.fields.jsonb.JSONField(null=True),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0007_project_overwrite_conflicts.py

# Generated by Django 2.2.17 on 2020-12-21 19:43

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0006_auto_20201214_1642"),
    ]

    operations = [
        migrations.AddField(
            model_name="project",
            name="overwrite_conflicts",
            field=models.BooleanField(default=True),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0008_update_site_name.py

# Generated by Django 2.2.17 on 2020-12-31 15:04

from django.conf import settings
from django.db import migrations


def update_site_name(apps, schema_editor):
    site_model = apps.get_model("sites", "Site")
    domain = "qfield.cloud"
    name = "QFieldCloud"

    site_model.objects.update_or_create(
        pk=settings.SITE_ID, defaults={"domain": domain, "name": name}
    )


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0007_project_overwrite_conflicts"),
        ("sites", "0002_alter_domain_unique"),
    ]

    operations = [
        migrations.RunPython(update_site_name),
    ]


# docker-app/qfieldcloud/core/migrations/0009_geodb.py

# Generated by Django 2.2.17 on 2021-01-03 11:41

import django.db.models.deletion
import qfieldcloud.core.models
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0008_update_site_name"),
    ]

    operations = [
        migrations.CreateModel(
            name="Geodb",
            fields=[
                (
                    "user",
                    models.OneToOneField(
                        on_delete=django.db.models.deletion.CASCADE,
                        primary_key=True,
                        serialize=False,
                        to=settings.AUTH_USER_MODEL,
                    ),
                ),
                (
                    "username",
                    models.CharField(
                        default=qfieldcloud.core.models.Geodb.random_string,
                        max_length=255,
                    ),
                ),
                (
                    "dbname",
                    models.CharField(
                        default=qfieldcloud.core.models.Geodb.random_string,
                        max_length=255,
                    ),
                ),
                (
                    "hostname",
                    models.CharField(
                        default=qfieldcloud.core.models.Geodb.default_hostname,
                        max_length=255,
                    ),
                ),
                (
                    "port",
                    models.PositiveIntegerField(
                        default=qfieldcloud.core.models.Geodb.default_port
                    ),
                ),
                ("created_at", models.DateTimeField(auto_now_add=True)),
            ],
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0010_auto_20210106_1543.py

# Generated by Django 2.2.17 on 2021-01-06 15:43

import django.core.validators
import qfieldcloud.core.validators
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0009_geodb"),
    ]

    operations = [
        migrations.AlterField(
            model_name="project",
            name="name",
            field=models.CharField(
                help_text="Project name",
                max_length=255,
                validators=[
                    django.core.validators.RegexValidator(
                        "^[-a-zA-Z0-9_]+$",
                        "Only letters, numbers, underscores or hyphens are allowed.",
                    ),
                    django.core.validators.RegexValidator(
                        "^.{3,}$", "The name must be at least 3 characters long."
                    ),
                    django.core.validators.RegexValidator(
                        "^[a-zA-Z].*$", "The name must begin with a letter."
                    ),
                    qfieldcloud.core.validators.reserved_words_validator,
                ],
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0011_export.py

# Generated by Django 2.2.17 on 2021-01-06 08:42

import uuid

import django.contrib.postgres.fields.jsonb
import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0010_auto_20210106_1543"),
    ]

    operations = [
        migrations.CreateModel(
            name="Export",
            fields=[
                (
                    "id",
                    models.UUIDField(
                        default=uuid.uuid4,
                        editable=False,
                        primary_key=True,
                        serialize=False,
                    ),
                ),
                (
                    "status",
                    models.PositiveSmallIntegerField(
                        choices=[
                            (1, "STATUS_PENDING"),
                            (2, "STATUS_BUSY"),
                            (3, "STATUS_EXPORTED"),
                            (4, "STATUS_ERROR"),
                        ],
                        default=1,
                    ),
                ),
                ("output", models.TextField(null=True)),
                (
                    "exportlog",
                    django.contrib.postgres.fields.jsonb.JSONField(null=True),
                ),
                ("created_at", models.DateTimeField(auto_now_add=True)),
                ("updated_at", models.DateTimeField(auto_now=True)),
                (
                    "project",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="exports",
                        to="core.Project",
                    ),
                ),
            ],
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0012_auto_20210106_0930.py

# Generated by Django 2.2.17 on 2021-01-06 09:30

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0011_export"),
    ]

    operations = [
        migrations.RenameModel(
            old_name="Export",
            new_name="Exportation",
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0013_remove_exportation_output.py

# Generated by Django 2.2.17 on 2021-01-12 09:47

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0012_auto_20210106_0930"),
    ]

    operations = [
        migrations.RemoveField(
            model_name="exportation",
            name="output",
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0014_exportation_output.py

# Generated by Django 2.2.17 on 2021-01-18 16:03

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0013_remove_exportation_output"),
    ]

    operations = [
        migrations.AddField(
            model_name="exportation",
            name="output",
            field=models.TextField(null=True),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0015_auto_20210123_0116.py

# Generated by Django 2.2.17 on 2021-01-23 01:16

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0014_exportation_output"),
    ]

    operations = [
        migrations.AddField(
            model_name="useraccount",
            name="bio",
            field=models.CharField(default="", max_length=255),
        ),
        migrations.AddField(
            model_name="useraccount",
            name="is_email_public",
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name="useraccount",
            name="location",
            field=models.CharField(default="", max_length=255),
        ),
        migrations.AddField(
            model_name="useraccount",
            name="twitter",
            field=models.CharField(default="", max_length=255),
        ),
        migrations.AddField(
            model_name="useraccount",
            name="workplace",
            field=models.CharField(default="", max_length=255),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0016_generate_user_account.py

from django.db import migrations


def generate_user_account(apps, schema_editor):
    User = apps.get_model("core", "User")
    UserAccount = apps.get_model("core", "UserAccount")

    for user in User.objects.filter(useraccount=None):
        UserAccount.objects.create(user=user)


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0015_auto_20210123_0116"),
    ]

    operations = [
        migrations.RunPython(
            generate_user_account, reverse_code=migrations.RunPython.noop
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0017_auto_20210226_1939.py

# Generated by Django 2.2.17 on 2021-02-26 19:39

import django.core.validators
import qfieldcloud.core.validators
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0016_generate_user_account"),
    ]

    operations = [
        migrations.AlterField(
            model_name="project",
            name="name",
            field=models.CharField(
                help_text="Project name. Should start with a letter and contain only letters, numbers, underscores and hyphens.",
                max_length=255,
                validators=[
                    django.core.validators.RegexValidator(
                        "^[-a-zA-Z0-9_]+$",
                        "Only letters, numbers, underscores or hyphens are allowed.",
                    ),
                    django.core.validators.RegexValidator(
                        "^.{3,}$", "The name must be at least 3 characters long."
                    ),
                    django.core.validators.RegexValidator(
                        "^[a-zA-Z].*$", "The name must begin with a letter."
                    ),
                    qfieldcloud.core.validators.reserved_words_validator,
                ],
            ),
        ),
        migrations.AlterField(
            model_name="project",
            name="overwrite_conflicts",
            field=models.BooleanField(
                default=True,
                help_text="If enabled, QFieldCloud will automatically overwrite conflicts in this project. Disabling this will force the project manager to manually resolve all the conflicts.",
            ),
        ),
        migrations.AlterField(
            model_name="project",
            name="private",
            field=models.BooleanField(
                default=False,
                help_text="Projects that are not marked as private would be visible and editable to anyone.",
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0018_auto_20210308_0034.py

# Generated by Django 2.2.17 on 2021-03-08 00:34

from django.db import migrations, models
from django.db.models.aggregates import Count


class Migration(migrations.Migration):
    def forwards_func(apps, schema_editor):
        Project = apps.get_model("core", "Project")

        nonunique_projects_data = (
            Project.objects.values("name", "owner")
            .annotate(c=Count("id"))
            .filter(c__gt=1)
        )

        for project_data in nonunique_projects_data:
            count = 0

            for project in Project.objects.filter(
                name=project_data.get("name"), owner_id=project_data.get("owner")
            ):
                count += 1

                if count == 1:
                    continue

                project.name += f"_{count}"
                project.save()

    dependencies = [
        ("core", "0017_auto_20210226_1939"),
    ]

    operations = [
        migrations.RunPython(forwards_func, migrations.RunPython.noop),
        migrations.AddConstraint(
            model_name="project",
            constraint=models.UniqueConstraint(
                fields=("owner", "name"), name="project_owner_name_uniq"
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0019_auto_20210316_2056.py

# Generated by Django 2.2.17 on 2021-03-16 20:56

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0018_auto_20210308_0034"),
    ]

    operations = [
        migrations.AlterField(
            model_name="delta",
            name="status",
            field=models.PositiveSmallIntegerField(
                choices=[
                    (1, "STATUS_PENDING"),
                    (2, "STATUS_BUSY"),
                    (3, "STATUS_APPLIED"),
                    (4, "STATUS_CONFLICT"),
                    (5, "STATUS_NOT_APPLIED"),
                    (6, "STATUS_ERROR"),
                    (7, "STATUS_IGNORED"),
                ],
                default=1,
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0020_auto_20210321_1749.py

# Generated by Django 2.2.17 on 2021-03-21 17:49

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0019_auto_20210316_2056"),
    ]

    operations = [
        migrations.AlterField(
            model_name="delta",
            name="status",
            field=models.PositiveSmallIntegerField(
                choices=[
                    (1, "STATUS_PENDING"),
                    (2, "STATUS_BUSY"),
                    (3, "STATUS_APPLIED"),
                    (4, "STATUS_CONFLICT"),
                    (5, "STATUS_NOT_APPLIED"),
                    (6, "STATUS_ERROR"),
                    (7, "STATUS_IGNORED"),
                    (8, "STATUS_UNPERMITTED"),
                ],
                default=1,
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0021_auto_20210330_2209.py

# Generated by Django 2.2.19 on 2021-03-30 22:09

from django.db import migrations, models
from django.db.models.aggregates import Count


class Migration(migrations.Migration):
    def forwards_func(apps, schema_editor):
        ProjectCollaborator = apps.get_model("core", "ProjectCollaborator")

        duplicated_collaborators_data = (
            ProjectCollaborator.objects.values("project", "collaborator", "role")
            .annotate(c=Count("id"))
            .filter(c__gt=1)
        )

        # this is catching only the very simple case of multiple collaborators that are also project owners
        # but NOT cases when the collaborator:
        # 1) is the organization owner of the project owner of type organization
        # 2) is the organization admin of the project owner of type organization
        for collaborators_data in duplicated_collaborators_data:
            queryset = ProjectCollaborator.objects.filter(
                project=collaborators_data["project"],
                collaborator=collaborators_data["collaborator"],
            ).order_by("role")

            queryset.exclude(pk=queryset.first().pk).delete()
            queryset.save()

        for project_collaborator in ProjectCollaborator.objects.all():
            collaborator = project_collaborator.collaborator
            project = project_collaborator.project

            if collaborator == project.owner:
                project_collaborator.delete()

    dependencies = [
        ("core", "0020_auto_20210321_1749"),
    ]

    operations = [
        migrations.RunPython(forwards_func, migrations.RunPython.noop),
        migrations.AddConstraint(
            model_name="projectcollaborator",
            constraint=models.UniqueConstraint(
                fields=("project", "collaborator"),
                name="projectcollaborator_project_collaborator_uniq",
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0022_auto_20210331_0143.py

# Generated by Django 2.2.19 on 2021-03-31 01:43

from django.db import migrations, models
from django.db.models.aggregates import Count


class Migration(migrations.Migration):
    def forwards_func(apps, schema_editor):
        OrganizationMember = apps.get_model("core", "OrganizationMember")

        duplicated_members_data = (
            OrganizationMember.objects.values("organization", "member")
            .annotate(c=Count("id"))
            .filter(c__gt=1)
        )

        for member_data in duplicated_members_data:
            queryset = OrganizationMember.objects.filter(
                organization=member_data["organization"],
                member=member_data["member"],
            ).order_by("role")

            queryset.exclude(pk=queryset.first().pk).delete()
            queryset.save()

        for organization_member in OrganizationMember.objects.all():
            member = organization_member.member
            organization = organization_member.organization

            if member == organization.organization_owner:
                organization_member.delete()

    dependencies = [
        ("core", "0021_auto_20210330_2209"),
    ]

    operations = [
        migrations.RunPython(forwards_func, migrations.RunPython.noop),
        migrations.AddConstraint(
            model_name="organizationmember",
            constraint=models.UniqueConstraint(
                fields=("organization", "member"),
                name="organization_organization_member_uniq",
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0023_auto_20210407_1247.py

# Generated by Django 2.2.19 on 2021-04-07 12:47

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0022_auto_20210331_0143"),
    ]

    operations = [
        migrations.AddField(
            model_name="user",
            name="remaining_invitations",
            field=models.PositiveIntegerField(default=3),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0024_auto_20210407_2002.py

# Generated by Django 2.2.19 on 2021-04-07 20:02

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0023_auto_20210407_1247"),
    ]

    operations = [
        migrations.AddField(
            model_name="user",
            name="is_geodb_enabled",
            field=models.BooleanField(
                default=True,
                help_text="Whether the account has the option to create a GeoDB.",
            ),
        ),
        migrations.AlterField(
            model_name="user",
            name="is_geodb_enabled",
            field=models.BooleanField(
                default=False,
                help_text="Whether the account has the option to create a GeoDB.",
            ),
        ),
        migrations.AlterField(
            model_name="user",
            name="remaining_invitations",
            field=models.PositiveIntegerField(
                default=3,
                help_text="Remaining invitations that can be sent by the user himself.",
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0025_auto_20210407_2306.py

# Generated by Django 3.1.8 on 2021-04-07 23:06

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0024_auto_20210407_2002"),
        ("auth", "0012_alter_user_first_name_max_length"),
    ]

    operations = [
        migrations.AlterField(
            model_name="delta",
            name="content",
            field=models.JSONField(),
        ),
        migrations.AlterField(
            model_name="delta",
            name="output",
            field=models.JSONField(null=True),
        ),
        migrations.AlterField(
            model_name="exportation",
            name="exportlog",
            field=models.JSONField(null=True),
        ),
        migrations.AlterField(
            model_name="user",
            name="first_name",
            field=models.CharField(
                blank=True, max_length=150, verbose_name="first name"
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0026_auto_20210409_1339.py

# Generated by Django 3.2 on 2021-04-09 13:39

from django.db import migrations, models


class Migration(migrations.Migration):
    def generate_user_account(apps, schema_editor):
        User = apps.get_model("core", "User")
        UserAccount = apps.get_model("core", "UserAccount")

        for user in User.objects.filter(useraccount=None):
            UserAccount.objects.create(user=user)

    def transfer_is_geodb_enabled_forwards(apps, schema_editor):
        User = apps.get_model("core", "User")

        for user in User.objects.all():
            user.useraccount.is_geodb_enabled = user.is_geodb_enabled

    def transfer_is_geodb_enabled_backwards(apps, schema_editor):
        User = apps.get_model("core", "User")

        for user in User.objects.all():
            user.is_geodb_enabled = user.useraccount.is_geodb_enabled

    dependencies = [
        ("core", "0025_auto_20210407_2306"),
    ]

    operations = [
        migrations.AddField(
            model_name="useraccount",
            name="is_geodb_enabled",
            field=models.BooleanField(
                default=False,
                help_text="Whether the account has the option to create a GeoDB.",
            ),
        ),
        migrations.RunPython(
            generate_user_account, reverse_code=migrations.RunPython.noop
        ),
        migrations.RunPython(
            transfer_is_geodb_enabled_forwards,
            reverse_code=transfer_is_geodb_enabled_backwards,
        ),
        migrations.RemoveField(
            model_name="user",
            name="is_geodb_enabled",
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0027_auto_20210408_0138.py

# Generated by Django 3.2 on 2021-04-08 01:38

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0026_auto_20210409_1339"),
    ]

    operations = [
        migrations.AddField(
            model_name="useraccount",
            name="avatar_uri",
            field=models.CharField(
                blank=True,
                max_length=255,
                verbose_name="Profile Picture URI",
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0028_auto_20210414_1545.py

# Generated by Django 3.2 on 2021-04-14 15:45

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0027_auto_20210408_0138"),
    ]

    def set_is_public(apps, schema_editor):
        Project = apps.get_model("core", "Project")

        for project in Project.objects.all():
            project.is_public = not project.private
            project.save()

    operations = [
        migrations.AddField(
            model_name="project",
            name="is_public",
            field=models.BooleanField(
                default=False,
                help_text="Projects that are marked as public would be visible and editable to anyone.",
            ),
        ),
        migrations.RunPython(set_is_public, reverse_code=migrations.RunPython.noop),
        migrations.RemoveField(
            model_name="project",
            name="private",
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0029_auto_20210415_1420.py

# Generated by Django 3.2 on 2021-04-15 14:20

import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0028_auto_20210414_1545"),
    ]

    operations = [
        migrations.AlterModelOptions(
            name="project",
            options={"ordering": ["owner__username", "name"]},
        ),
        migrations.AlterField(
            model_name="project",
            name="owner",
            field=models.ForeignKey(
                on_delete=django.db.models.deletion.CASCADE,
                related_name="projects",
                to=settings.AUTH_USER_MODEL,
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0030_refactor_roles.py

# Generated by Django 3.2 on 2021-04-15 11:59

from django.db import migrations, models


def refactor_roles(apps, schema_editor):
    OrganizationMember = apps.get_model("core", "OrganizationMember")
    ProjectCollaborator = apps.get_model("core", "ProjectCollaborator")
    OrganizationMember.objects.filter(role=1).update(role_str="admin")
    OrganizationMember.objects.filter(role=2).update(role_str="member")
    ProjectCollaborator.objects.filter(role=1).update(role_str="admin")
    ProjectCollaborator.objects.filter(role=2).update(role_str="manager")
    ProjectCollaborator.objects.filter(role=3).update(role_str="editor")
    ProjectCollaborator.objects.filter(role=4).update(role_str="reporter")
    ProjectCollaborator.objects.filter(role=5).update(role_str="reader")


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0029_auto_20210415_1420"),
    ]

    operations = [
        migrations.AddField(
            model_name="organizationmember",
            name="role_str",
            field=models.CharField(
                choices=[("admin", "Admin"), ("member", "Member")],
                default="member",
                max_length=10,
            ),
        ),
        migrations.AddField(
            model_name="projectcollaborator",
            name="role_str",
            field=models.CharField(
                choices=[
                    ("admin", "Admin"),
                    ("manager", "Manager"),
                    ("editor", "Editor"),
                    ("reporter", "Reporter"),
                    ("reader", "Reader"),
                ],
                default="reader",
                max_length=10,
            ),
        ),
        migrations.RunPython(refactor_roles),
        migrations.RemoveField(
            model_name="organizationmember",
            name="role",
        ),
        migrations.RemoveField(
            model_name="projectcollaborator",
            name="role",
        ),
        migrations.RenameField(
            model_name="organizationmember",
            old_name="role_str",
            new_name="role",
        ),
        migrations.RenameField(
            model_name="projectcollaborator",
            old_name="role_str",
            new_name="role",
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0031_auto_20210415_1535.py

# Generated by Django 3.2 on 2021-04-15 15:35

import django.contrib.auth.models
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0030_refactor_roles"),
    ]

    operations = [
        migrations.CreateModel(
            name="Team",
            fields=[
                (
                    "user_ptr",
                    models.OneToOneField(
                        auto_created=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        parent_link=True,
                        primary_key=True,
                        serialize=False,
                        to="core.user",
                    ),
                ),
                (
                    "team_organization",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="teams",
                        to="core.organization",
                    ),
                ),
            ],
            options={
                "verbose_name": "team",
                "verbose_name_plural": "teams",
            },
            bases=("core.user",),
            managers=[
                ("objects", django.contrib.auth.models.UserManager()),
            ],
        ),
        migrations.AlterField(
            model_name="user",
            name="user_type",
            field=models.PositiveSmallIntegerField(
                choices=[(1, "user"), (2, "organization"), (3, "team")],
                default=1,
                editable=False,
            ),
        ),
        migrations.CreateModel(
            name="TeamMember",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "member",
                    models.ForeignKey(
                        limit_choices_to=models.Q(user_type=1),
                        on_delete=django.db.models.deletion.CASCADE,
                        to=settings.AUTH_USER_MODEL,
                    ),
                ),
                (
                    "team",
                    models.ForeignKey(
                        limit_choices_to=models.Q(user_type=3),
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="members",
                        to="core.team",
                    ),
                ),
            ],
        ),
        migrations.AddConstraint(
            model_name="teammember",
            constraint=models.UniqueConstraint(
                fields=("team", "member"), name="team_team_member_uniq"
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0032_auto_20210419_1011.py

# Generated by Django 3.2 on 2021-04-19 10:11

import django.db.models.deletion
import qfieldcloud.core.models
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0031_auto_20210415_1535"),
    ]

    operations = [
        migrations.AlterModelManagers(
            name="organization",
            managers=[
                ("objects", qfieldcloud.core.models.OrganizationManager()),
            ],
        ),
        migrations.AddField(
            model_name="organizationmember",
            name="is_public",
            field=models.BooleanField(default=False),
        ),
        migrations.AlterField(
            model_name="project",
            name="owner",
            field=models.ForeignKey(
                limit_choices_to=models.Q(user_type__in=[1, 2]),
                on_delete=django.db.models.deletion.CASCADE,
                related_name="projects",
                to=settings.AUTH_USER_MODEL,
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0033_auto_20210419_1908.py

# Generated by Django 3.2 on 2021-04-19 19:08

import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):
    def add_created_by(apps, schema_editor):
        Delta = apps.get_model("core", "Delta")

        for delta in Delta.objects.all():
            delta.created_by = delta.project.owner
            delta.save()

    dependencies = [
        ("core", "0032_auto_20210419_1011"),
    ]

    operations = [
        migrations.AddField(
            model_name="delta",
            name="created_by",
            field=models.ForeignKey(
                on_delete=django.db.models.deletion.CASCADE,
                related_name="uploaded_deltas",
                to="core.user",
                null=True,
            ),
        ),
        migrations.RunPython(add_created_by, migrations.RunPython.noop),
        migrations.AlterField(
            model_name="delta",
            name="created_by",
            field=models.ForeignKey(
                on_delete=django.db.models.deletion.CASCADE,
                related_name="uploaded_deltas",
                to="core.user",
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0034_auto_20210508_1030.py

# Generated by Django 3.2 on 2021-05-08 10:30

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0033_auto_20210419_1908"),
    ]

    operations = [
        migrations.AddField(
            model_name="user",
            name="has_accepted_tos",
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name="user",
            name="has_newsletter_subscription",
            field=models.BooleanField(default=False),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0035_auto_20210510_0635.py

# Generated by Django 3.2.2 on 2021-05-10 06:35

import django.core.validators
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0034_auto_20210508_1030"),
    ]

    operations = [
        migrations.AlterField(
            model_name="project",
            name="name",
            field=models.CharField(
                help_text="Only letters, numbers, underscores, hyphens and dots are allowed.",
                max_length=255,
                validators=[
                    django.core.validators.RegexValidator(
                        "^[a-zA-Z0-9-_\\.]+$",
                        "Only letters, numbers, underscores, hyphens and dots are allowed.",
                    )
                ],
            ),
        ),
        migrations.AlterField(
            model_name="useraccount",
            name="bio",
            field=models.CharField(blank=True, default="", max_length=255),
        ),
        migrations.AlterField(
            model_name="useraccount",
            name="location",
            field=models.CharField(blank=True, default="", max_length=255),
        ),
        migrations.AlterField(
            model_name="useraccount",
            name="twitter",
            field=models.CharField(blank=True, default="", max_length=255),
        ),
        migrations.AlterField(
            model_name="useraccount",
            name="workplace",
            field=models.CharField(blank=True, default="", max_length=255),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0036_auto_20210426_1210.py

# Generated by Django 3.2 on 2021-04-26 12:10

import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):
    def refactor_status_forward(apps, schema_editor):
        Job = apps.get_model("core", "Job")
        Job.objects.filter(status=1).update(status_str="queued")
        Job.objects.filter(status=2).update(status_str="deferred")
        Job.objects.filter(status=3).update(status_str="finished")
        Job.objects.filter(status=4).update(status_str="failed")

    def refactor_status_backward(apps, schema_editor):
        Job = apps.get_model("core", "Job")
        Job.objects.filter(status_str="queued").update(status=1)
        Job.objects.filter(status_str="started").update(status=2)
        Job.objects.filter(status_str="deferred").update(status=2)
        Job.objects.filter(status_str="stopped").update(status=2)
        Job.objects.filter(status_str="finished").update(status=3)
        Job.objects.filter(status_str="failed").update(status=4)

    def add_created_by(apps, schema_editor):
        Job = apps.get_model("core", "Job")

        for job in Job.objects.all():
            job.created_by = job.project.owner
            job.save()

    def add_export_jobs_forward(apps, schema_editor):
        Job = apps.get_model("core", "Job")
        ExportJob = apps.get_model("core", "ExportJob")

        for job in Job.objects.all():
            ExportJob.objects.create(pk=job.pk, exportlog=job.exportlog)

    def add_export_jobs_backward(apps, schema_editor):
        Job = apps.get_model("core", "Job")
        ExportJob = apps.get_model("core", "ExportJob")

        for export_job in ExportJob.objects.all():
            Job.objects.filter(pk=export_job.pk).update(exportlog=export_job.exportlog)

    def refactor_deltas_status_forward(apps, schema_editor):
        Delta = apps.get_model("core", "Delta")

        Delta.objects.filter(status=1).update(last_status="pending")
        Delta.objects.filter(status=2).update(last_status="started")
        Delta.objects.filter(status=3).update(last_status="applied")
        Delta.objects.filter(status=4).update(last_status="conflict")
        Delta.objects.filter(status=5).update(last_status="not_applied")
        Delta.objects.filter(status=6).update(last_status="error")
        Delta.objects.filter(status=7).update(last_status="ignored")
        Delta.objects.filter(status=8).update(last_status="unpermitted")

    def refactor_deltas_status_backward(apps, schema_editor):
        Delta = apps.get_model("core", "Delta")

        Delta.objects.filter(last_status="pending").update(status=1)
        Delta.objects.filter(last_status="started").update(status=2)
        Delta.objects.filter(last_status="applied").update(status=3)
        Delta.objects.filter(last_status="conflict").update(status=4)
        Delta.objects.filter(last_status="not_applied").update(status=5)
        Delta.objects.filter(last_status="error").update(status=6)
        Delta.objects.filter(last_status="ignored").update(status=7)
        Delta.objects.filter(last_status="unpermitted").update(status=8)

    def refactor_deltas_forward(apps, schema_editor):
        Delta = apps.get_model("core", "Delta")
        ApplyJob = apps.get_model("core", "ApplyJob")
        ApplyJobDelta = apps.get_model("core", "ApplyJobDelta")

        ApplyJob.objects.all().delete()

        for delta in Delta.objects.all():
            apply_job = ApplyJob.objects.create(
                project=delta.project,
                created_by=delta.created_by,
                overwrite_conflicts=True,
                status="finished",
                type="delta_apply",
            )

            ApplyJobDelta.objects.create(
                apply_job=apply_job,
                delta=delta,
                status=delta.last_status,
                feedback=delta.last_feedback,
            )

    dependencies = [
        ("core", "0035_auto_20210510_0635"),
    ]

    operations = [
        migrations.RenameModel(
            old_name="Exportation",
            new_name="Job",
        ),
        migrations.AddField(
            model_name="job",
            name="type",
            field=models.CharField(
                choices=[("export", "Export"), ("delta_apply", "Delta Apply")],
                max_length=32,
                default="export",
            ),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name="job",
            name="status_str",
            field=models.CharField(
                choices=[
                    ("pending", "Pending"),
                    ("queued", "Queued"),
                    ("started", "Started"),
                    ("finished", "Finished"),
                    ("stopped", "Stopped"),
                    ("failed", "Failed"),
                ],
                default="pending",
                max_length=32,
            ),
            preserve_default=False,
        ),
        migrations.RunPython(refactor_status_forward, refactor_status_backward),
        migrations.RemoveField(
            model_name="job",
            name="status",
        ),
        migrations.RenameField(
            model_name="job",
            old_name="status_str",
            new_name="status",
        ),
        migrations.AddField(
            model_name="job",
            name="created_by",
            field=models.ForeignKey(
                on_delete=django.db.models.deletion.CASCADE, to="core.user", null=True
            ),
            preserve_default=False,
        ),
        migrations.RunPython(add_created_by, migrations.RunPython.noop),
        migrations.AlterField(
            model_name="job",
            name="created_by",
            field=models.ForeignKey(
                on_delete=django.db.models.deletion.CASCADE, to="core.user"
            ),
        ),
        # no idea why, but it is needed
        migrations.AlterField(
            model_name="job",
            name="status",
            field=models.CharField(
                choices=[
                    ("pending", "Pending"),
                    ("queued", "Queued"),
                    ("started", "Started"),
                    ("finished", "Finished"),
                    ("stopped", "Stopped"),
                    ("failed", "Failed"),
                ],
                default="queued",
                max_length=32,
            ),
        ),
        migrations.CreateModel(
            name="ExportJob",
            fields=[
                (
                    "job_ptr",
                    models.OneToOneField(
                        auto_created=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        parent_link=True,
                        primary_key=True,
                        serialize=False,
                        to="core.job",
                    ),
                ),
                (
                    "exportlog",
                    models.JSONField(null=True),
                ),
            ],
        ),
        migrations.RunPython(add_export_jobs_forward, add_export_jobs_backward),
        migrations.RemoveField(
            model_name="job",
            name="exportlog",
        ),
        migrations.CreateModel(
            name="ApplyJob",
            fields=[
                (
                    "job_ptr",
                    models.OneToOneField(
                        auto_created=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        parent_link=True,
                        primary_key=True,
                        serialize=False,
                        to="core.job",
                    ),
                ),
            ],
            bases=("core.job",),
        ),
        migrations.AddField(
            model_name="applyjob",
            name="overwrite_conflicts",
            field=models.BooleanField(
                default=True,
                help_text="If enabled, QFieldCloud will automatically overwrite conflicts while applying deltas.",
            ),
            preserve_default=False,
        ),
        migrations.CreateModel(
            name="ApplyJobDelta",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "status",
                    models.CharField(
                        choices=[
                            ("pending", "Pending"),
                            ("started", "Started"),
                            ("applied", "Applied"),
                            ("conflict", "Conflict"),
                            ("not_applied", "Not_applied"),
                            ("error", "Error"),
                            ("ignored", "Ignored"),
                            ("unpermitted", "Unpermitted"),
                        ],
                        default="pending",
                        max_length=32,
                    ),
                ),
                ("feedback", models.JSONField(null=True)),
                (
                    "delta",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="core.delta"
                    ),
                ),
                (
                    "apply_job",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="core.applyjob",
                    ),
                ),
            ],
        ),
        migrations.AddField(
            model_name="applyjob",
            name="deltas_to_apply",
            field=models.ManyToManyField(through="core.ApplyJobDelta", to="core.Delta"),
        ),
        migrations.RenameField(
            model_name="delta",
            old_name="output",
            new_name="last_feedback",
        ),
        migrations.AddField(
            model_name="delta",
            name="last_status",
            field=models.CharField(
                choices=[
                    ("pending", "Pending"),
                    ("started", "Started"),
                    ("applied", "Applied"),
                    ("conflict", "Conflict"),
                    ("not_applied", "Not_applied"),
                    ("error", "Error"),
                    ("ignored", "Ignored"),
                    ("unpermitted", "Unpermitted"),
                ],
                default="pending",
                max_length=32,
            ),
        ),
        migrations.RunPython(
            refactor_deltas_status_forward, refactor_deltas_status_backward
        ),
        migrations.RemoveField(
            model_name="delta",
            name="status",
        ),
        migrations.RunSQL("SET CONSTRAINTS ALL IMMEDIATE;", migrations.RunSQL.noop),
        migrations.RunPython(refactor_deltas_forward, migrations.RunPython.noop),
        migrations.RunSQL("SET CONSTRAINTS ALL DEFERRED;", migrations.RunSQL.noop),
    ]


# docker-app/qfieldcloud/core/migrations/0037_alter_project_owner_help_text.py

# Generated by Django 3.2.2 on 2021-05-17 09:36

import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0036_auto_20210426_1210"),
    ]

    operations = [
        migrations.AlterField(
            model_name="project",
            name="owner",
            field=models.ForeignKey(
                help_text="The project owner can be either you or any of the organization you are member of.",
                limit_choices_to=models.Q(("user_type__in", [1, 2])),
                on_delete=django.db.models.deletion.CASCADE,
                related_name="projects",
                to=settings.AUTH_USER_MODEL,
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0038_rename_workplace_useraccount_company.py

# Generated by Django 3.2.2 on 2021-05-17 10:57

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0037_alter_project_owner_help_text"),
    ]

    operations = [
        migrations.RenameField(
            model_name="useraccount",
            old_name="workplace",
            new_name="company",
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0039_auto_20210630_1210.py

# Generated by Django 3.2.4 on 2021-06-30 12:10

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0038_rename_workplace_useraccount_company"),
    ]

    operations = [
        migrations.AlterModelOptions(
            name="applyjob",
            options={
                "verbose_name": "Job: apply",
                "verbose_name_plural": "Jobs: apply",
            },
        ),
        migrations.AlterModelOptions(
            name="exportjob",
            options={
                "verbose_name": "Job: export",
                "verbose_name_plural": "Jobs: export",
            },
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0040_auto_20210630_1212.py

# Generated by Django 3.2.4 on 2021-06-30 12:12

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0039_auto_20210630_1210"),
    ]

    operations = [
        migrations.AddField(
            model_name="delta",
            name="last_modified_pk",
            field=models.TextField(null=True),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0041_auto_20210705_0837.py

# Generated by Django 3.2.5 on 2021-07-05 08:37

from django.db import migrations, models
from qfieldcloud.core.utils import get_qgis_project_file


class Migration(migrations.Migration):
    def forwards_job_feedback_func(apps, schema_editor):
        Job = apps.get_model("core", "Job")
        ExportJob = apps.get_model("core", "ExportJob")
        ProcessProjectfileJob = apps.get_model("core", "ProcessProjectfileJob")

        for export_job in ExportJob.objects.all():
            job = Job.objects.get(id=export_job.job_ptr_id)
            job.feedback = {
                "legacy": {
                    "exportlog": export_job.exportlog,
                }
            }
            job.save()

        for process_projectfile_job in ProcessProjectfileJob.objects.all():
            job = Job.objects.get(id=process_projectfile_job.job_ptr_id)
            job.feedback = process_projectfile_job.feedback_old
            job.save()

    def backwards_job_feedback_func(apps, schema_editor):
        Job = apps.get_model("core", "Job")
        ExportJob = apps.get_model("core", "ExportJob")
        ProcessProjectfileJob = apps.get_model("core", "ProcessProjectfileJob")

        for export_job in ExportJob.objects.all():
            job = Job.objects.get(id=export_job.job_ptr_id)

            if job.feedback and job.feedback.get("legacy", {}).get("exportlog"):
                export_job.exportlog = job.feedback.get("legacy").get("exportlog")
                export_job.save()

        for process_projectfile_job in ProcessProjectfileJob.objects.all():
            job = Job.objects.get(id=process_projectfile_job.job_ptr_id)
            process_projectfile_job.feedback_old = job.feedback
            process_projectfile_job.save()

    def forwards_find_project_filename_func(apps, schema_editor):
        Project = apps.get_model("core", "Project")

        for project in Project.objects.all():
            project.project_filename = get_qgis_project_file(project.id)
            project.save()

    dependencies = [
        ("core", "0040_auto_20210630_1212"),
    ]

    operations = [
        migrations.AddField(
            model_name="project",
            name="project_filename",
            field=models.TextField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name="project",
            name="thumbnail_uri",
            field=models.CharField(
                blank=True, max_length=255, verbose_name="Thumbnail Picture URI"
            ),
        ),
        migrations.AddField(
            model_name="applyjobdelta",
            name="modified_pk",
            field=models.TextField(null=True),
        ),
        migrations.CreateModel(
            name="ProcessProjectfileJob",
            fields=[
                (
                    "job_ptr",
                    models.OneToOneField(
                        auto_created=True,
                        on_delete=models.deletion.CASCADE,
                        parent_link=True,
                        primary_key=True,
                        serialize=False,
                        to="core.job",
                    ),
                ),
                ("feedback", models.JSONField(null=True)),
            ],
            options={
                "verbose_name": "Job: process QGIS project file",
                "verbose_name_plural": "Jobs: process QGIS project file",
            },
            bases=("core.job",),
        ),
        migrations.AlterField(
            model_name="job",
            name="type",
            field=models.CharField(
                choices=[
                    ("export", "Export"),
                    ("delta_apply", "Delta Apply"),
                    ("process_projectfile", "Process QGIS Project File"),
                ],
                max_length=32,
            ),
        ),
        migrations.AlterField(
            model_name="job",
            name="status",
            field=models.CharField(
                choices=[
                    ("pending", "Pending"),
                    ("queued", "Queued"),
                    ("started", "Started"),
                    ("finished", "Finished"),
                    ("stopped", "Stopped"),
                    ("failed", "Failed"),
                ],
                default="pending",
                max_length=32,
            ),
        ),
        # add feedback field for all jobs
        migrations.RenameField(
            model_name="processprojectfilejob",
            old_name="feedback",
            new_name="feedback_old",
        ),
        migrations.AddField(
            model_name="job",
            name="feedback",
            field=models.JSONField(null=True),
        ),
        migrations.RunPython(forwards_job_feedback_func, backwards_job_feedback_func),
        migrations.RemoveField(
            model_name="exportjob",
            name="exportlog",
        ),
        migrations.RemoveField(
            model_name="processprojectfilejob",
            name="feedback_old",
        ),
        # / add feedback field for all jobs
        # / add feedback field for all jobs
        migrations.RunPython(
            forwards_find_project_filename_func, migrations.RunPython.noop
        ),
        # rename project.exports to project.jobs
        migrations.AlterField(
            model_name="job",
            name="project",
            field=models.ForeignKey(
                on_delete=models.deletion.CASCADE,
                related_name="jobs",
                to="core.project",
            ),
        ),
        # / rename project.exports to project.jobs
    ]


# docker-app/qfieldcloud/core/migrations/0042_auto_20211001_2217.py

# Generated by Django 3.2.7 on 2021-10-01 22:17

import qfieldcloud.core.models
from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0041_auto_20210705_0837"),
    ]

    operations = [
        migrations.AlterModelManagers(
            name="team",
            managers=[
                ("objects", qfieldcloud.core.models.QFieldCloudUserManager()),
            ],
        ),
        migrations.AlterModelManagers(
            name="user",
            managers=[
                ("objects", qfieldcloud.core.models.QFieldCloudUserManager()),
            ],
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0043_update_site_name.py

# Generated by Django 3.2.7 on 2021-10-05 22:17

import os

from django.conf import settings
from django.db import migrations


def update_site_name(apps, schema_editor):
    site_model = apps.get_model("sites", "Site")
    domain = os.environ.get("QFIELDCLOUD_HOST")
    name = "QFieldCloud"

    site_model.objects.update_or_create(
        pk=settings.SITE_ID, defaults={"domain": domain, "name": name}
    )


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0042_auto_20211001_2217"),
        ("sites", "0002_alter_domain_unique"),
    ]

    operations = [
        migrations.RunPython(update_site_name),
    ]


# docker-app/qfieldcloud/core/migrations/0044_alter_user_username.py

# Generated by Django 3.2.8 on 2021-10-06 13:38

import django.core.validators
import qfieldcloud.core.validators
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0043_update_site_name"),
    ]

    operations = [
        migrations.AlterField(
            model_name="user",
            name="username",
            field=models.CharField(
                error_messages={"unique": "A user with that username already exists."},
                help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.",
                max_length=150,
                unique=True,
                validators=[
                    django.core.validators.RegexValidator(
                        "^[-a-zA-Z0-9_]+$",
                        "Only letters, numbers, underscores or hyphens are allowed.",
                    ),
                    django.core.validators.RegexValidator(
                        "^[a-zA-Z].*$", "The name must begin with a letter."
                    ),
                    django.core.validators.RegexValidator(
                        "^.{3,}$", "The name must be at least 3 characters long."
                    ),
                    qfieldcloud.core.validators.reserved_words_validator,
                ],
                verbose_name="username",
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0045_auto_20211012_2234.py

# Generated by Django 3.2.8 on 2021-10-12 22:34

from django.db import migrations, models


def fill_in_datetime_fields(apps, schema_editor):
    # Old values in output field of delta table where string instead of json
    Job = apps.get_model("core", "Job")
    Job.objects.update(
        started_at=models.F("created_at"), finished_at=models.F("updated_at")
    )


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0044_alter_user_username"),
    ]

    operations = [
        migrations.AddField(
            model_name="job",
            name="finished_at",
            field=models.DateTimeField(blank=True, null=True, editable=False),
        ),
        migrations.AddField(
            model_name="job",
            name="started_at",
            field=models.DateTimeField(blank=True, null=True, editable=False),
        ),
        migrations.RunPython(fill_in_datetime_fields, migrations.RunPython.noop),
    ]


# docker-app/qfieldcloud/core/migrations/0046_auto_20211013_2304.py

# Generated by Django 3.2.8 on 2021-10-13 23:04

from django.db import migrations, models


def set_project_details(apps, schema_editor):
    # Old values in output field of delta table where string instead of json
    Project = apps.get_model("core", "Project")
    ProcessProjectfileJob = apps.get_model("core", "ProcessProjectfileJob")

    for project in Project.objects.filter(project_filename__isnull=False):
        ProcessProjectfileJob.objects.create(
            project=project,
            created_by=project.owner,
            type="process_projectfile",
        )


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0045_auto_20211012_2234"),
    ]

    operations = [
        migrations.AddField(
            model_name="project",
            name="project_details",
            field=models.JSONField(blank=True, null=True),
        ),
        migrations.RunPython(set_project_details, migrations.RunPython.noop),
    ]


# docker-app/qfieldcloud/core/migrations/0047_useraccount_timezone.py

# Generated by Django 3.2.8 on 2021-10-20 21:35

import timezone_field.fields
from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0046_auto_20211013_2304"),
    ]

    operations = [
        migrations.AddField(
            model_name="useraccount",
            name="timezone",
            field=timezone_field.fields.TimeZoneField(
                choices_display="WITH_GMT_OFFSET", default="Europe/Zurich"
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0048_useraccount_notifs_frequency.py

# Generated by Django 3.2.8 on 2021-10-18 09:52

import datetime

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0047_useraccount_timezone"),
    ]

    operations = [
        migrations.AddField(
            model_name="useraccount",
            name="notifs_frequency",
            field=models.DurationField(
                blank=True,
                choices=[
                    (datetime.timedelta(0), "Immediately"),
                    (datetime.timedelta(seconds=3600), "Hourly"),
                    (datetime.timedelta(days=1), "Daily"),
                    (datetime.timedelta(days=7), "Weekly"),
                    (None, "Disabled"),
                ],
                default=None,
                null=True,
                verbose_name="Email frequency for notifications",
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0049_auto_20211117_1843.py

# Generated by Django 3.2.9 on 2021-11-17 17:43

import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models


def fill_in_last_apply_attempt_at(apps, schema_editor):
    # Old values in output field of delta table where string instead of json
    Delta = apps.get_model("core", "Delta")
    ApplyJobDelta = apps.get_model("core", "ApplyJobDelta")

    for delta in Delta.objects.all():
        jobs_qs = ApplyJobDelta.objects.filter(delta=delta)

        if jobs_qs.count():
            job_delta = jobs_qs.latest("apply_job__started_at")
            delta.last_apply_attempt_at = job_delta.apply_job.started_at
            delta.last_apply_attempt_by = job_delta.apply_job.created_by
            delta.save()


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0048_useraccount_notifs_frequency"),
    ]

    operations = [
        migrations.AddField(
            model_name="delta",
            name="last_apply_attempt_at",
            field=models.DateTimeField(null=True),
        ),
        migrations.AddField(
            model_name="delta",
            name="last_apply_attempt_by",
            field=models.ForeignKey(
                null=True,
                on_delete=django.db.models.deletion.CASCADE,
                to=settings.AUTH_USER_MODEL,
            ),
        ),
        migrations.RunPython(fill_in_last_apply_attempt_at, migrations.RunPython.noop),
    ]


# docker-app/qfieldcloud/core/migrations/0050_auto_20211118_1150.py

# Generated by Django 3.2.8 on 2021-10-27 09:50

from django.db import migrations, models


def rename_export_to_package(apps, schema_editor):
    Job = apps.get_model("core", "Job")
    Job.objects.filter(type="export").update(type="package")


def rename_package_to_export(apps, schema_editor):
    Job = apps.get_model("core", "Job")
    Job.objects.filter(type="package").update(type="export")


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0049_auto_20211117_1843"),
    ]

    operations = [
        migrations.AddField(
            model_name="project",
            name="data_last_packaged_at",
            field=models.DateTimeField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name="project",
            name="data_last_updated_at",
            field=models.DateTimeField(blank=True, null=True),
        ),
        migrations.RenameModel(
            old_name="ExportJob",
            new_name="PackageJob",
        ),
        migrations.AlterField(
            model_name="job",
            name="type",
            field=models.CharField(
                choices=[
                    ("package", "Package"),
                    ("delta_apply", "Delta Apply"),
                    ("process_projectfile", "Process QGIS Project File"),
                ],
                max_length=32,
            ),
        ),
        migrations.RunPython(rename_export_to_package, rename_package_to_export),
    ]


# docker-app/qfieldcloud/core/migrations/0051_auto_20211125_0444.py

# Generated by Django 3.2.9 on 2021-11-25 03:44

import django.contrib.gis.db.models.fields
from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0050_auto_20211118_1150"),
    ]

    operations = [
        migrations.AlterModelOptions(
            name="packagejob",
            options={
                "verbose_name": "Job: package",
                "verbose_name_plural": "Jobs: package",
            },
        ),
        migrations.AddField(
            model_name="delta",
            name="new_geom",
            field=django.contrib.gis.db.models.fields.GeometryField(
                dim=4, null=True, srid=4326
            ),
        ),
        migrations.AddField(
            model_name="delta",
            name="old_geom",
            field=django.contrib.gis.db.models.fields.GeometryField(
                dim=4, null=True, srid=4326
            ),
        ),
        migrations.RunSQL(
            r"""
                WITH subquery AS (
                    SELECT
                        id,
                        CASE
                            WHEN jsonb_extract_path_text(content, 'localLayerCrs') ~ '^EPSG:\d{1,10}$'
                            THEN
                                REGEXP_REPLACE(jsonb_extract_path_text(content, 'localLayerCrs'), '\D*', '', 'g')::int
                            ELSE
                                NULL
                        END AS srid
                    FROM core_delta
                )
                UPDATE core_delta
                SET
                    old_geom =
                        ST_Transform(
                            ST_SetSRID(
                                ST_Force2D(
                                    ST_GeomFromText(
                                        REPLACE( jsonb_extract_path_text(core_delta.content, 'old', 'geometry'), 'nan', '0' )
                                    )
                                ),
                                subquery.srid
                            ),
                            4326
                        ),
                    new_geom =
                        ST_Transform(
                            ST_SetSRID(
                                ST_Force2D(
                                    ST_GeomFromText(
                                        REPLACE( jsonb_extract_path_text(core_delta.content, 'new', 'geometry'), 'nan', '0' )
                                    )
                                ),
                                subquery.srid
                            ),
                            4326
                        )
                FROM subquery
                WHERE core_delta.id = subquery.id
            """,
            migrations.RunSQL.noop,
        ),
        migrations.RunSQL(
            r"""
                CREATE FUNCTION core_delta_geom_trigger_func()
                RETURNS trigger
                AS
                $$
                    DECLARE
                        srid int;
                    BEGIN
                        SELECT CASE
                            WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\d{1,10}$'
                            THEN
                                REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\D*', '', 'g')::int
                            ELSE
                                NULL
                            END INTO srid;
                        NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );
                        NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );
                        RETURN NEW;
                    END;
                $$
                LANGUAGE PLPGSQL
            """,
            r"""
                DROP FUNCTION core_delta_geom_trigger_func();
            """,
        ),
        migrations.RunSQL(
            r"""
                CREATE TRIGGER core_delta_geom_update_trigger BEFORE UPDATE ON core_delta
                FOR EACH ROW
                WHEN (OLD.content IS DISTINCT FROM NEW.content)
                EXECUTE FUNCTION core_delta_geom_trigger_func()
            """,
            r"""
                DROP TRIGGER core_delta_geom_update_trigger ON core_delta;
            """,
        ),
        migrations.RunSQL(
            r"""
                CREATE TRIGGER core_delta_geom_insert_trigger BEFORE INSERT ON core_delta
                FOR EACH ROW
                EXECUTE FUNCTION core_delta_geom_trigger_func()
            """,
            r"""
                DROP TRIGGER core_delta_geom_insert_trigger ON core_delta
            """,
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0052_secret.py

# Generated by Django 3.2.12 on 2022-04-13 14:44

import django.core.validators
import django.db.models.deletion
import django_cryptography.fields
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0051_auto_20211125_0444"),
    ]

    operations = [
        migrations.CreateModel(
            name="Secret",
            fields=[
                (
                    "id",
                    models.AutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "name",
                    models.TextField(
                        help_text="Must start with a capital letter and followed by capital letters, numbers or underscores.",
                        max_length=255,
                        unique=True,
                        validators=[
                            django.core.validators.RegexValidator(
                                "^[A-Z]+[A-Z0-9_]+$",
                                "Must start with a capital letter and followed by capital letters, numbers or underscores.",
                            )
                        ],
                    ),
                ),
                (
                    "type",
                    models.CharField(
                        choices=[
                            ("pgservice", "pg_service"),
                            ("envvar", "Environment Variable"),
                        ],
                        max_length=32,
                    ),
                ),
                (
                    "project",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="secrets",
                        to="core.project",
                    ),
                ),
                ("created_at", models.DateTimeField(auto_now_add=True)),
                (
                    "created_by",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        related_name="project_secrets",
                        to="core.user",
                    ),
                ),
                ("value", django_cryptography.fields.encrypt(models.TextField())),
            ],
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0053_auto_20220505_1948.py

# Generated by Django 3.2.13 on 2022-05-05 17:48

import django.core.validators
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0052_secret"),
    ]

    operations = [
        migrations.AlterModelOptions(
            name="secret",
            options={"ordering": ["project", "name"]},
        ),
        migrations.AlterField(
            model_name="secret",
            name="name",
            field=models.TextField(
                help_text="Must start with a capital letter and followed by capital letters, numbers or underscores.",
                max_length=255,
                validators=[
                    django.core.validators.RegexValidator(
                        "^[A-Z]+[A-Z0-9_]+$",
                        "Must start with a capital letter and followed by capital letters, numbers or underscores.",
                    )
                ],
            ),
        ),
        migrations.AddConstraint(
            model_name="secret",
            constraint=models.UniqueConstraint(
                fields=("project", "name"), name="secret_project_name_uniq"
            ),
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0054_project_last_package.py

# Generated by Django 3.2.12 on 2022-03-09 15:58

import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("core", "0053_auto_20220505_1948"),
    ]

    operations = [
        migrations.AddField(
            model_name="project",
            name="last_package_job",
            field=models.ForeignKey(
                blank=True,
                null=True,
                on_delete=django.db.models.deletion.SET_NULL,
                related_name="last_job_of",
                to="core.packagejob",
            ),
        ),
        migrations.AddField(
            model_name="delta",
            name="jobs_to_apply",
            field=models.ManyToManyField(
                through="core.ApplyJobDelta", to="core.ApplyJob"
            ),
        ),
        migrations.RunSQL(
            sql=r'CREATE UNIQUE INDEX "core_user_username_uppercase" ON "core_user" (UPPER("username"));',
            reverse_sql=r'DROP INDEX IF EXISTS "core_user_username_uppercase";',
        ),
    ]


# docker-app/qfieldcloud/core/migrations/0055_auto_20220204_1654.py

# Generated by Django 3.2.11 on 2022-02-08 20:59

import datetime

import django.core.validators
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ("subscription", "0002_populate_plans"),
        ("core", "0054_project_last_package"),
    ]

    operations = [
        migrations.RemoveField(
            model_name="useraccount",
            name="storage_limit_mb",
        ),
        migrations.RemoveField(
            model_name="useraccount",
            name="synchronizations_per_months",
        ),
        migrations.AddField(
            model_name="project",
            name="repackaging_cache_expire",
            field=models.DurationField(
                default=datetime.timedelta(seconds=3600),
                validators=[
                    django.core.validators.MinValueValidator(
                        datetime.timedelta(seconds=60)
                    )
                ],
            ),
        ),
        migrations.RenameField(
            model_name="useraccount",
            old_name="account_type",
            new_name="plan",
        ),
        migrations.AlterField(
            model_name="useraccount",
            name="plan",
            field=models.ForeignKey(
                on_delete=django.db.models.deletion.PROTECT,
                to="subscription.plan",
            ),
        ),
        migrations.AddField(
            model_name="project",
            name="storage_size_mb",
            field=models.FloatField(default=0),
        ),
        migrations.AlterField(
            model_name="projectcollaborator",
            name="collaborator",
            field=models.ForeignKey(
                limit_choices_to=models.Q(("user_type__in", [1, 3])),
                on_delete=django.db.models.deletion.CASCADE,
                to=settings.AUTH_USER_MODEL,
            ),
        ),
    ]


# docker-app/qfieldcloud/core/utils2/__init__.py

import qfieldcloud.core.utils2.audit as audit
import qfieldcloud.core.utils2.jobs as jobs
import qfieldcloud.core.utils2.storage as storage

__all__ = ["audit", "jobs", "storage"]


# docker-app/qfieldcloud/core/utils2/audit.py

import json
from typing import Any, Dict, List, Union

from auditlog.models import LogEntry
from django.contrib.auth.models import AnonymousUser, User
from django_currentuser.middleware import get_current_authenticated_user


def audit(
    instance,
    action: LogEntry.Action,
    changes: Union[Dict[str, Any], List[Any], str] = None,
    actor: User = None,
    remote_addr: str = None,
    additional_data: Any = None,
):
    changes_json = None

    try:
        if changes is not None:
            changes_json = json.dumps(changes)
    except Exception:
        changes_json = json.dumps(str(changes))

    if actor is None:
        actor = get_current_authenticated_user()
    elif isinstance(actor, AnonymousUser):
        actor = None

    actor_id = actor.pk if actor else None

    return LogEntry.objects.log_create(
        instance,
        action=action,
        changes=changes_json,
        actor_id=actor_id,
        remote_addr=remote_addr,
        additional_data=additional_data,
    )


# docker-app/qfieldcloud/core/utils2/jobs.py

import logging
from typing import List

import qfieldcloud.core.models as models
from django.conf import settings
from django.db import transaction
from django.db.models import Q
from qfieldcloud.core import exceptions

logger = logging.getLogger(__name__)


@transaction.atomic
def apply_deltas(
    project: "models.Project",
    user: "models.User",
    project_file: str,
    overwrite_conflicts: bool,
    delta_ids: List[str] = [],
) -> List["models.ApplyJob"]:
    """Apply a deltas"""

    logger.info(
        f"Requested apply_deltas on {project} with {project_file}; overwrite_conflicts: {overwrite_conflicts}; delta_ids: {delta_ids}"
    )

    # 1. Check if there are any pending deltas.
    # We need to call .select_for_update() to make sure there would not be a concurrent
    # request that will try to apply these deltas.
    pending_deltas = models.Delta.objects.select_for_update().filter(
        project=project,
        last_status=models.Delta.Status.PENDING,
    )

    # 1.1. Filter only the deltas of interest.
    if len(delta_ids) > 0:
        pending_deltas.filter(pk__in=delta_ids)

    # 2. If there are no pending deltas, do not create a new job and return.
    if pending_deltas.count() == 0:
        return None

    # 3. Find all the pending or queued jobs in the queue.
    # If an "apply_delta" job is in a "started" status, we don't know how far the execution reached
    # so we better assume the deltas will reach a non-"pending" status.
    apply_jobs = models.ApplyJob.objects.filter(
        project=project,
        status=[
            models.Job.Status.PENDING,
            models.Job.Status.QUEUED,
        ],
    )

    # 4. Check whether there are jobs found in the queue and exclude all deltas that are part of any pending job.
    if apply_jobs.count() > 0:
        pending_deltas = pending_deltas.exclude(jobs_to_apply__in=apply_jobs)

    # 5. If there are no pending deltas, do not create a new job and return.
    deltas_count = pending_deltas.count()
    if deltas_count == 0:
        return None

    # 6. There are pending deltas that are not part of any pending job. So we create one.
    apply_jobs = []
    for i in range(deltas_count // settings.APPLY_DELTAS_LIMIT + 1):
        offset = settings.APPLY_DELTAS_LIMIT * i
        limit = max(settings.APPLY_DELTAS_LIMIT * (i + 1), deltas_count)

        if offset == limit:
            break

        apply_job = models.ApplyJob.objects.create(
            project=project,
            created_by=user,
            overwrite_conflicts=overwrite_conflicts,
        )

        models.ApplyJobDelta.objects.bulk_create(
            [
                models.ApplyJobDelta(
                    apply_job=apply_job,
                    delta=delta,
                )
                for delta in pending_deltas.order_by("created_at")[offset:limit]
            ]
        )

        apply_jobs.append(apply_job)

    # 7. return the created job
    return apply_jobs


def repackage(project: "models.Project", user: "models.User") -> "models.PackageJob":
    """Returns an unfinished or freshly created package job.

    Checks if there is already an unfinished package job and returns it,
    or creates a new package job and returns it.
    """
    if not project.project_filename:
        raise exceptions.NoQGISProjectError()

    # Check if active package job already exists
    query = Q(project=project) & (
        Q(status=models.PackageJob.Status.PENDING)
        | Q(status=models.PackageJob.Status.QUEUED)
        | Q(status=models.PackageJob.Status.STARTED)
    )

    if models.PackageJob.objects.filter(query).count():
        return models.PackageJob.objects.get(query)

    package_job = models.PackageJob.objects.create(project=project, created_by=user)

    return package_job


def repackage_if_needed(
    project: "models.Project", user: "models.User"
) -> "models.PackageJob":
    if not project.project_filename:
        raise exceptions.NoQGISProjectError()

    if project.needs_repackaging:
        package_job = repackage(project, user)
    else:
        package_job = (
            models.PackageJob.objects.filter(project=project)
            .order_by("started_at")
            .get()
        )

    return package_job


# docker-app/qfieldcloud/core/utils2/projects.py

from typing import Tuple

from django.db.models import Q
from django.utils.translation import gettext as _
from qfieldcloud.core import invitations_utils as invitation
from qfieldcloud.core import permissions_utils as perms
from qfieldcloud.core.models import Project, ProjectCollaborator, User


def create_collaborator(project: Project, user: User) -> Tuple[bool, str]:
    """Creates a new collaborator (qfieldcloud.core.ProjectCollaborator) if possible

    Args:
        project (Project): the project to add collaborator to
        user (User): the user to be added as collaborator

    Returns:
        Tuple[bool, str]: success, message - whether the collaborator creation was success and explanation message of the outcome
    """
    success, message = False, ""

    try:
        perms.check_can_become_collaborator(user, project)

        # TODO actual invitation, not insertion of already existing user (see TODO about inviting
        # users to create accounts above)
        ProjectCollaborator.objects.create(
            project=project,
            collaborator=user,
        )
        success = True
        message = _('User "{}" has been invited to the project.').format(user.username)
    except perms.UserOrganizationRoleError:
        message = _(
            "User '{}' is not a member of the organization that owns the project. "
            "Please add this user to the organization first."
        ).format(user.username)
    except (
        perms.AlreadyCollaboratorError,
        perms.ReachedCollaboratorLimitError,
        perms.ExpectedPremiumUserError,
    ) as err:
        message = str(err)

    return success, message


def create_collaborator_by_username_or_email(
    project: Project, username: str, created_by: User
):
    """Creates a new collaborator (qfieldcloud.core.ProjectCollaborator) if possible

    Args:
        project (Project): the project to add collaborator to
        user (str): the username or email to be added as collaborator or invited to join QFieldCloud
        created_by (User): the user that initiated the collaborator creation

    Returns:
        Tuple[bool, str]: success, message - whether the collaborator creation was success and explanation message of the outcome
    """
    success, message = False, ""
    users = list(
        User.objects.filter(Q(username=username) | Q(email=username)).exclude(
            user_type=User.TYPE_ORGANIZATION
        )
    )

    if len(users) == 0:
        # No user found, if string is an email address, we try to send a link
        if invitation.is_valid_email(username):
            success, message = invitation.invite_user_by_email(username, created_by)
        else:
            message = _('User "{}" does not exist.').format(username)
    elif len(users) > 1:
        # TODO support invitation of multiple users
        message = _("Adding multiple collaborators at once is not supported.").format(
            username
        )
    elif users[0].is_organization:
        message = (
            _(
                'Organization "{}" cannot be added. Only users and teams can be collaborators.'
            ).format(username),
        )
    else:
        success, message = create_collaborator(project, users[0])

    return success, message


# docker-app/qfieldcloud/core/utils2/storage.py

from __future__ import annotations

import logging
import os
from pathlib import PurePath
from typing import IO, List, Set

import qfieldcloud.core.models
import qfieldcloud.core.utils
from django.conf import settings
from django.core.files.base import ContentFile
from django.http import FileResponse, HttpRequest
from django.http.response import HttpResponse, HttpResponseBase
from qfieldcloud.core.utils2.audit import LogEntry, audit

logger = logging.getLogger(__name__)

QFIELDCLOUD_HOST = os.environ.get("QFIELDCLOUD_HOST", None)
WEB_HTTPS_PORT = os.environ.get("WEB_HTTPS_PORT", None)


def get_attachment_dir_prefix(project: "Project", filename: str) -> str:  # noqa: F821
    """Returns the attachment dir where the file belongs to or empty string if it does not.

    Args:
        project (Project): project to check
        filename (str): filename to check

    Returns:
        str: the attachment dir or empty string if no match found
    """
    for attachment_dir in project.attachment_dirs:
        if filename.startswith(attachment_dir):
            return attachment_dir

    return ""


def file_response(
    request: HttpRequest,
    key: str,
    presigned: bool = False,
    expires: int = 60,
    version: str = None,
    as_attachment: bool = False,
) -> HttpResponseBase:
    url = ""
    filename = PurePath(key).name
    extra_params = {}

    if version is not None:
        extra_params["VersionId"] = version

    # check if we are in NGINX proxy
    http_host = request.META.get("HTTP_HOST", "")
    https_port = http_host.split(":")[-1] if ":" in http_host else "443"

    if https_port == WEB_HTTPS_PORT and not settings.IN_TEST_SUITE:
        if presigned:
            if as_attachment:
                extra_params["ResponseContentType"] = "application/force-download"
                extra_params[
                    "ResponseContentDisposition"
                ] = f'attachment;filename="{filename}"'

            url = qfieldcloud.core.utils.get_s3_client().generate_presigned_url(
                "get_object",
                Params={
                    **extra_params,
                    "Key": key,
                    "Bucket": qfieldcloud.core.utils.get_s3_bucket().name,
                },
                ExpiresIn=expires,
                HttpMethod="GET",
            )
        else:
            url = qfieldcloud.core.utils.get_s3_object_url(key)

        # Let's NGINX handle the redirect to the storage and streaming the file contents back to the client
        response = HttpResponse()
        response["X-Accel-Redirect"] = "/storage-download/"
        response["redirect_uri"] = url

        return response
    elif settings.DEBUG or settings.IN_TEST_SUITE:
        return_file = ContentFile(b"")
        qfieldcloud.core.utils.get_s3_bucket().download_fileobj(
            key,
            return_file,
            extra_params,
        )

        return FileResponse(
            return_file.open(),
            as_attachment=as_attachment,
            filename=filename,
            content_type="text/html",
        )

    raise Exception(
        "Expected to either run behind nginx proxy, debug mode or within a test suite."
    )


def upload_user_avatar(user: "User", file: IO, mimetype: str) -> str:  # noqa: F821
    """Uploads a picture as a user avatar.

    NOTE this function does NOT modify the `UserAccount.avatar_uri` field

    Args:
        user (User):
        file (IO): file used as avatar
        mimetype (str): file mimetype

    Returns:
        str: URI to the avatar
    """
    bucket = qfieldcloud.core.utils.get_s3_bucket()

    if mimetype == "image/svg+xml":
        extension = "svg"
    elif mimetype == "image/png":
        extension = "png"
    elif mimetype == "image/jpeg":
        extension = "jpg"
    else:
        raise Exception(f"Unknown mimetype: {mimetype}")

    key = f"users/{user.username}/avatar.{extension}"
    bucket.upload_fileobj(
        file,
        key,
        {
            "ACL": "public-read",
            "ContentType": mimetype,
        },
    )
    return key


def remove_user_avatar(user: "User") -> None:  # noqa: F821
    """Removes the user's avatar file.

    NOTE this function does NOT modify the `UserAccount.avatar_uri` field

    Args:
        user (User):
    """
    bucket = qfieldcloud.core.utils.get_s3_bucket()
    key = user.useraccount.avatar_uri
    bucket.object_versions.filter(Prefix=key).delete()


def upload_project_thumbail(
    project: "Project", file: IO, mimetype: str, filename: str  # noqa: F821
) -> str:
    """Uploads a picture as a project thumbnail.

    NOTE this function does NOT modify the `Project.thumbnail_uri` field

    Args:
        project (Project):
        file (IO): file used as thumbail
        mimetype (str): file mimetype
        filename (str): filename

    Returns:
        str: URI to the thumbnail
    """
    bucket = qfieldcloud.core.utils.get_s3_bucket()

    # for now we always expect PNGs
    if mimetype == "image/svg+xml":
        extension = "svg"
    elif mimetype == "image/png":
        extension = "png"
    elif mimetype == "image/jpeg":
        extension = "jpg"
    else:
        raise Exception(f"Unknown mimetype: {mimetype}")

    key = f"projects/{project.id}/meta/{filename}.{extension}"
    bucket.upload_fileobj(
        file,
        key,
        {
            # TODO most probably this is not public-read, since the project might be private
            "ACL": "public-read",
            "ContentType": mimetype,
        },
    )
    return key


def remove_project_thumbail(project: "Project") -> None:  # noqa: F821
    """Uploads a picture as a project thumbnail.

    NOTE this function does NOT modify the `Project.thumbnail_uri` field

    """
    bucket = qfieldcloud.core.utils.get_s3_bucket()
    key = project.thumbnail_uri
    bucket.object_versions.filter(Prefix=key).delete()


def purge_old_file_versions(project: "Project") -> None:  # noqa: F821
    """
    Deletes old versions of all files in the given project. Will keep __3__
    versions for COMMUNITY user accounts, and __10__ versions for PRO user
    accounts
    """

    logger.info(f"Cleaning up old files for {project}")

    # Number of versions to keep is determined by the account type
    keep_count = project.owner.useraccount.plan.storage_keep_versions

    logger.debug(f"Keeping {keep_count} versions")

    # Process file by file
    for file in qfieldcloud.core.utils.get_project_files_with_versions(project.pk):

        # Skip the newest N
        old_versions_to_purge = sorted(
            file.versions, key=lambda v: v.last_modified, reverse=True
        )[keep_count:]

        # Debug print
        logger.debug(
            f'Purging {len(old_versions_to_purge)} out of {len(file.versions)} old versions for "{file.latest.name}"...'
        )

        # Remove the N oldest
        for old_version in old_versions_to_purge:
            if old_version.is_latest:
                # This is not supposed to happen, as versions were sorted above,
                # but leaving it here as a security measure in case version
                # ordering changes for some reason.
                raise Exception("Trying to delete latest version")
            # TODO: any way to batch those ? will probaby get slow on production
            old_version._data.delete()
            # TODO: audit ? take implementation from files_views.py:211

    # Update the project size
    project.save(recompute_storage=True)


def delete_file(project: "Project", filename: str):  # noqa: F821
    file = qfieldcloud.core.utils.get_project_file_with_versions(project.id, filename)

    if not file:
        raise Exception("No file with such name in the given project found")

    file.delete()

    if qfieldcloud.core.utils.is_qgis_project_file(filename):
        project.project_filename = None
        project.save(recompute_storage=True)

    audit(
        project,
        LogEntry.Action.DELETE,
        changes={f"{filename} ALL": [file.latest.e_tag, None]},
    )


def delete_file_version(
    project: "Project",  # noqa: F821
    filename: str,
    version_id: str,
    include_older: bool = False,
) -> List[qfieldcloud.core.utils.S3ObjectVersion]:
    """Deletes a specific version of given file.

    Args:
        project (Project): project the file belongs to
        filename (str): filename the version belongs to
        version_id (str): version id to delete
        include_older (bool, optional): when True, versions older than the passed `version` will also be deleted. If the version_id is the latest version of a file, this parameter will treated as False. Defaults to False.

    Returns:
        int: the number of versions deleted
    """
    file = qfieldcloud.core.utils.get_project_file_with_versions(project.id, filename)

    if not file:
        raise Exception("No file with such name in the given project found")

    is_deleting_all_versions = False
    if file.latest.id == version_id:
        include_older = False

        if len(file.versions) == 1:
            is_deleting_all_versions = True

    versions_to_delete: List[qfieldcloud.core.utils.S3ObjectVersion] = []

    for file_version in file.versions:
        if file_version.id == version_id:
            versions_to_delete.append(file_version)

            if include_older:
                continue
            else:
                break

        if versions_to_delete:
            assert (
                include_older
            ), "We should continue to loop only if `include_older` is True"
            assert (
                versions_to_delete[-1].last_modified > file_version.last_modified
            ), "Assert the other versions are really older than the requested one"

            versions_to_delete.append(file_version)

    for file_version in versions_to_delete:
        file_version._data.delete()

        audit_suffix = "ALL" if is_deleting_all_versions else file_version.display

        audit(
            project,
            LogEntry.Action.DELETE,
            changes={f"{filename} {audit_suffix}": [file_version.e_tag, None]},
        )

    if is_deleting_all_versions and qfieldcloud.core.utils.is_qgis_project_file(
        filename
    ):
        project.project_filename = None
        project.save(recompute_storage=True)

    return versions_to_delete


def get_stored_package_ids(project_id: str) -> Set[str]:
    bucket = qfieldcloud.core.utils.get_s3_bucket()
    prefix = f"projects/{project_id}/packages/"
    root_path = PurePath(prefix)
    package_ids = set()

    for file in bucket.objects.filter(Prefix=prefix):
        file_path = PurePath(file.key)
        parts = file_path.relative_to(root_path).parts
        package_ids.add(parts[0])

    return package_ids


def delete_stored_package(project_id: str, package_id: str) -> None:
    bucket = qfieldcloud.core.utils.get_s3_bucket()
    prefix = f"projects/{project_id}/packages/{package_id}/"

    bucket.objects.filter(Prefix=prefix).delete()


# docker-app/qfieldcloud/core/views/collaborators_views.py

from django.contrib.auth import get_user_model
from django.core.exceptions import ObjectDoesNotExist
from django.utils.decorators import method_decorator
from drf_yasg.utils import swagger_auto_schema
from qfieldcloud.core import permissions_utils
from qfieldcloud.core.models import Project, ProjectCollaborator
from qfieldcloud.core.serializers import ProjectCollaboratorSerializer
from rest_framework import generics, permissions, status
from rest_framework.response import Response
from rest_framework.settings import api_settings

User = get_user_model()


class ListCreateCollaboratorsViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        user = request.user
        projectid = permissions_utils.get_param_from_request(request, "projectid")
        try:
            project = Project.objects.get(id=projectid)
        except ObjectDoesNotExist:
            return False

        if request.method == "GET":
            return permissions_utils.can_read_collaborators(user, project)
        if request.method == "POST":
            return permissions_utils.can_create_collaborators(user, project)
        return False


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="List collaborators of the project",
        operation_id="List collaborators",
    ),
)
@method_decorator(
    name="post",
    decorator=swagger_auto_schema(
        operation_description="Add a user as collaborator of the project",
        operation_id="Create collaborator",
    ),
)
class ListCreateCollaboratorsView(generics.ListCreateAPIView):

    permission_classes = [
        permissions.IsAuthenticated,
        ListCreateCollaboratorsViewPermissions,
    ]
    serializer_class = ProjectCollaboratorSerializer

    def get_queryset(self):

        project_id = self.request.parser_context["kwargs"]["projectid"]
        project_obj = Project.objects.get(id=project_id)

        return ProjectCollaborator.objects.filter(project=project_obj)

    def post(self, request, projectid):

        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        collaborator = User.objects.get(username=request.data["collaborator"])
        project = Project.objects.get(id=projectid)
        serializer.save(collaborator=collaborator, project=project)

        try:
            headers = {"Location": str(serializer.data[api_settings.URL_FIELD_NAME])}
        except (TypeError, KeyError):
            headers = {}

        return Response(
            serializer.data, status=status.HTTP_201_CREATED, headers=headers
        )


class GetUpdateDestroyCollaboratorViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        user = request.user
        projectid = permissions_utils.get_param_from_request(request, "projectid")

        try:
            project = Project.objects.get(id=projectid)
        except ObjectDoesNotExist:
            return False

        if request.method == "GET":
            return permissions_utils.can_read_collaborators(user, project)
        if request.method in ["PUT", "PATCH"]:
            return permissions_utils.can_update_collaborators(user, project)
        if request.method in ["DELETE"]:
            return permissions_utils.can_delete_collaborators(user, project)
        return False


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="Get the role of a collaborator",
        operation_id="Get collaborator",
    ),
)
@method_decorator(
    name="put",
    decorator=swagger_auto_schema(
        operation_description="Update a collaborator",
        operation_id="Update collaborator",
    ),
)
@method_decorator(
    name="patch",
    decorator=swagger_auto_schema(
        operation_description="Partial update collaborator",
        operation_id="Patch collaborator",
    ),
)
@method_decorator(
    name="delete",
    decorator=swagger_auto_schema(
        operation_description="Remove a collaborator from a project",
        operation_id="Delete collaborator",
    ),
)
class GetUpdateDestroyCollaboratorView(generics.RetrieveUpdateDestroyAPIView):

    permission_classes = [
        permissions.IsAuthenticated,
        GetUpdateDestroyCollaboratorViewPermissions,
    ]
    serializer_class = ProjectCollaboratorSerializer

    def get_object(self):
        project_id = self.request.parser_context["kwargs"]["projectid"]
        collaborator = self.request.parser_context["kwargs"]["username"]

        project_obj = Project.objects.get(id=project_id)
        collaborator_obj = User.objects.get(username=collaborator)
        return ProjectCollaborator.objects.get(
            project=project_obj, collaborator=collaborator_obj
        )


# docker-app/qfieldcloud/core/views/deltas_views.py

import json
import logging
from datetime import datetime

from django.contrib.auth import get_user_model
from django.db import transaction
from django.utils.decorators import method_decorator
from django.utils.translation import gettext as _
from drf_yasg.utils import swagger_auto_schema
from qfieldcloud.core import exceptions, permissions_utils, utils
from qfieldcloud.core.models import Delta, Project
from qfieldcloud.core.serializers import DeltaSerializer
from qfieldcloud.core.utils2 import jobs
from rest_framework import generics, permissions, views
from rest_framework.response import Response

User = get_user_model()

logger = logging.getLogger(__name__)


class DeltaFilePermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        projectid = permissions_utils.get_param_from_request(request, "projectid")
        project = Project.objects.get(id=projectid)
        user = request.user

        if request.method == "GET":
            return permissions_utils.can_read_deltas(user, project)
        if request.method == "POST":
            return permissions_utils.can_create_deltas(user, project)
        return False


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="List all deltas of a project",
        operation_id="List deltas",
    ),
)
@method_decorator(
    name="post",
    decorator=swagger_auto_schema(
        operation_description="Add a deltafile to a project",
        operation_id="Add deltafile",
    ),
)
class ListCreateDeltasView(generics.ListCreateAPIView):

    permission_classes = [permissions.IsAuthenticated, DeltaFilePermissions]
    serializer_class = DeltaSerializer

    def post(self, request, projectid):

        project_obj = Project.objects.get(id=projectid)
        project_file = project_obj.project_filename

        if "file" not in request.data:
            raise exceptions.EmptyContentError()

        request_file = utils.strip_json_null_bytes(request.data["file"])
        created_deltas = []

        try:
            deltafile_json = json.load(request_file)
            utils.get_deltafile_schema_validator().validate(deltafile_json)

            deltafile_id = deltafile_json["id"]
            deltafile_projectid = deltafile_json["project"]

            deltas = deltafile_json.get("deltas", [])
            delta_ids = sorted([str(delta["uuid"]) for delta in deltas])
            existing_delta_ids = [
                str(v)
                for v in Delta.objects.filter(id__in=delta_ids)
                .order_by("id")
                .values_list("id", flat=True)
            ]

            if project_file is None:
                raise exceptions.NoQGISProjectError()

            if deltafile_projectid != str(projectid):
                exc = exceptions.DeltafileValidationError()
                exc.message = f"Deltafile's project id ({deltafile_projectid}) doesn't match URL parameter project id ({project_obj.id})."
                raise exc

            with transaction.atomic():
                for delta in deltas:
                    if delta["uuid"] in existing_delta_ids:
                        logger.warning(f'Duplicate delta id: ${delta["uuid"]}')
                        continue

                    delta_obj = Delta(
                        id=delta["uuid"],
                        deltafile_id=deltafile_id,
                        project=project_obj,
                        content=delta,
                        created_by=self.request.user,
                    )

                    if not permissions_utils.can_create_delta(
                        self.request.user, delta_obj
                    ):
                        delta_obj.last_status = Delta.Status.UNPERMITTED
                        delta_obj.last_feedback = {
                            "msg": _(
                                "User has no rights to create delta on this project. Try inviting him as a collaborator with proper permissions and try again."
                            )
                        }
                    elif not delta_obj.is_supported_regarding_owner_account:
                        delta_obj.last_status = Delta.Status.UNPERMITTED
                        delta_obj.last_feedback = {
                            "msg": _(
                                "Some features of this project are not supported by the owner's account. Either upgrade the account or ensure you're not using features such as remote layers, then try again."
                            )
                        }
                    else:
                        delta_obj.last_status = Delta.Status.PENDING

                    delta_obj.save(force_insert=True)
                    created_deltas.append(delta_obj)

        except Exception as err:
            if request_file:
                key = f"projects/{projectid}/deltas/{datetime.now().isoformat()}.json"
                # otherwise we upload an empty file
                request_file.seek(0)
                utils.get_s3_bucket().upload_fileobj(request_file, key)
                logger.info(f'Invalid deltafile saved as "{key}"')

            logger.exception(err)

            if isinstance(err, exceptions.NoQGISProjectError):
                raise err
            elif isinstance(err, exceptions.DeltafileValidationError):
                raise err
            else:
                raise exceptions.QFieldCloudException() from err

        if created_deltas and not jobs.apply_deltas(
            project_obj,
            self.request.user,
            project_file,
            project_obj.overwrite_conflicts,
        ):
            logger.warning("Failed to start delta apply job.")

        return Response()

    def get_queryset(self):
        project_id = self.request.parser_context["kwargs"]["projectid"]
        project_obj = Project.objects.get(id=project_id)
        return Delta.objects.filter(project=project_obj)


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="List deltas of a deltafile",
        operation_id="List deltas of deltafile",
    ),
)
class ListDeltasByDeltafileView(generics.ListAPIView):

    permission_classes = [permissions.IsAuthenticated, DeltaFilePermissions]
    serializer_class = DeltaSerializer

    def get_queryset(self):
        project_id = self.request.parser_context["kwargs"]["projectid"]
        project_obj = Project.objects.get(id=project_id)
        deltafile_id = self.request.parser_context["kwargs"]["deltafileid"]
        return Delta.objects.filter(project=project_obj, deltafile_id=deltafile_id)


@method_decorator(
    name="post",
    decorator=swagger_auto_schema(
        operation_description="Trigger apply delta",
        operation_id="Apply delta",
    ),
)
class ApplyView(views.APIView):

    permission_classes = [permissions.IsAuthenticated, DeltaFilePermissions]
    serializer_class = DeltaSerializer

    def post(self, request, projectid):
        project_obj = Project.objects.get(id=projectid)
        project_file = project_obj.project_filename

        if project_file is None:
            raise exceptions.NoQGISProjectError()

        if not jobs.apply_deltas(
            project_obj,
            self.request.user,
            project_file,
            project_obj.overwrite_conflicts,
        ):
            logger.warning("Failed to start delta apply job.")

        return Response()


# docker-app/qfieldcloud/core/views/files_views.py

from pathlib import PurePath

import qfieldcloud.core.utils2 as utils2
from django.db import transaction
from django.utils import timezone
from qfieldcloud.core import exceptions, permissions_utils, utils
from qfieldcloud.core.models import Job, ProcessProjectfileJob, Project
from qfieldcloud.core.utils import S3ObjectVersion, get_project_file_with_versions
from qfieldcloud.core.utils2.audit import LogEntry, audit
from qfieldcloud.core.utils2.storage import (
    get_attachment_dir_prefix,
    purge_old_file_versions,
)
from rest_framework import permissions, status, views
from rest_framework.parsers import MultiPartParser
from rest_framework.response import Response


class ListFilesViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        if "projectid" not in request.parser_context["kwargs"]:
            return False

        projectid = request.parser_context["kwargs"]["projectid"]
        project = Project.objects.get(id=projectid)

        return permissions_utils.can_read_files(request.user, project)


class ListFilesView(views.APIView):
    # TODO: swagger doc
    # TODO: docstring

    permission_classes = [permissions.IsAuthenticated, ListFilesViewPermissions]

    def get(self, request, projectid):

        project = Project.objects.get(id=projectid)

        bucket = utils.get_s3_bucket()

        prefix = "projects/{}/files/".format(projectid)

        files = {}
        for version in bucket.object_versions.filter(Prefix=prefix):
            # Created the dict entry if doesn't exist
            if version.key not in files:
                files[version.key] = {"versions": []}

            head = version.head()
            path = PurePath(version.key)
            filename = str(path.relative_to(*path.parts[:3]))
            last_modified = version.last_modified.strftime("%d.%m.%Y %H:%M:%S %Z")

            # We cannot be sure of the metadata's first letter case
            # https://github.com/boto/boto3/issues/1709
            metadata = head["Metadata"]
            if "sha256sum" in metadata:
                sha256sum = metadata["sha256sum"]
            else:
                sha256sum = metadata["Sha256sum"]

            if version.is_latest:
                is_attachment = get_attachment_dir_prefix(project, filename) != ""

                files[version.key]["name"] = filename
                files[version.key]["size"] = version.size
                files[version.key]["sha256"] = sha256sum
                files[version.key]["md5sum"] = version.e_tag.replace('"', "")
                files[version.key]["last_modified"] = last_modified
                files[version.key]["is_attachment"] = is_attachment

            files[version.key]["versions"].append(
                {
                    "size": version.size,
                    "sha256": sha256sum,
                    "md5sum": version.e_tag.replace('"', ""),
                    "version_id": version.version_id,
                    "last_modified": last_modified,
                    "is_latest": version.is_latest,
                    "display": S3ObjectVersion(version.key, version).display,
                }
            )

        result_list = [files[key] for key in files]
        return Response(result_list)


class DownloadPushDeleteFileViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        if "projectid" not in request.parser_context["kwargs"]:
            return False

        projectid = request.parser_context["kwargs"]["projectid"]
        project = Project.objects.get(id=projectid)
        user = request.user

        if request.method == "GET":
            return permissions_utils.can_read_files(user, project)
        if request.method == "DELETE":
            return permissions_utils.can_delete_files(user, project)
        if request.method == "POST":
            return permissions_utils.can_create_files(user, project)
        return False


class DownloadPushDeleteFileView(views.APIView):
    # TODO: swagger doc
    # TODO: docstring
    parser_classes = [MultiPartParser]
    permission_classes = [
        permissions.IsAuthenticated,
        DownloadPushDeleteFileViewPermissions,
    ]

    def get(self, request, projectid, filename):
        Project.objects.get(id=projectid)

        version = None
        if "version" in self.request.query_params:
            version = self.request.query_params["version"]

        key = utils.safe_join("projects/{}/files/".format(projectid), filename)
        return utils2.storage.file_response(
            request,
            key,
            presigned=True,
            expires=600,
            version=version,
            as_attachment=True,
        )

    def post(self, request, projectid, filename, format=None):
        project = Project.objects.get(id=projectid)

        if "file" not in request.data:
            raise exceptions.EmptyContentError()

        is_qgis_project_file = utils.is_qgis_project_file(filename)
        # check only one qgs/qgz file per project
        if (
            is_qgis_project_file
            and project.project_filename is not None
            and PurePath(filename) != PurePath(project.project_filename)
        ):
            raise exceptions.MultipleProjectsError(
                "Only one QGIS project per project allowed"
            )

        request_file = request.FILES.get("file")

        file_size_mb = request_file.size / 1024 / 1024
        quota_left_mb = project.owner.useraccount.storage_quota_left_mb

        if file_size_mb > quota_left_mb:
            raise exceptions.QuotaError(
                f"Requiring {file_size_mb}MB of storage but only {quota_left_mb}MB available."
            )

        old_object = get_project_file_with_versions(project.id, filename)
        sha256sum = utils.get_sha256(request_file)
        bucket = utils.get_s3_bucket()

        key = utils.safe_join(f"projects/{projectid}/files/", filename)
        metadata = {"Sha256sum": sha256sum}

        bucket.upload_fileobj(request_file, key, ExtraArgs={"Metadata": metadata})

        new_object = get_project_file_with_versions(project.id, filename)

        assert new_object

        with transaction.atomic():
            # we only enter a transaction after the file is uploaded above because we do not
            # want to lock the project row for way too long. If we reselect for update the
            # project and update it now, it guarantees there will be no other file upload editing
            # the same project row.
            project = Project.objects.select_for_update().get(id=projectid)
            update_fields = ["data_last_updated_at"]

            if get_attachment_dir_prefix(project, filename) == "" and (
                is_qgis_project_file or project.project_filename is not None
            ):
                if is_qgis_project_file:
                    project.project_filename = filename
                    update_fields.append("project_filename")

                running_jobs = ProcessProjectfileJob.objects.filter(
                    project=project,
                    created_by=self.request.user,
                    status__in=[
                        Job.Status.PENDING,
                        Job.Status.QUEUED,
                        Job.Status.STARTED,
                    ],
                )

                if running_jobs.count() == 0:
                    ProcessProjectfileJob.objects.create(
                        project=project, created_by=self.request.user
                    )

            project.data_last_updated_at = timezone.now()
            project.save(update_fields=update_fields)

        if old_object:
            audit(
                project,
                LogEntry.Action.UPDATE,
                changes={filename: [old_object.latest.e_tag, new_object.latest.e_tag]},
            )
        else:
            audit(
                project,
                LogEntry.Action.CREATE,
                changes={filename: [None, new_object.latest.e_tag]},
            )

        # Delete the old file versions
        purge_old_file_versions(project)

        return Response(status=status.HTTP_201_CREATED)

    def delete(self, request, projectid, filename):
        project = Project.objects.get(id=projectid)
        version_id = request.META.get("HTTP_X_FILE_VERSION")

        if version_id:
            utils2.storage.delete_file_version(project, filename, version_id, False)
        else:
            utils2.storage.delete_file(project, filename)

        return Response(status=status.HTTP_200_OK)


class ProjectMetafilesView(views.APIView):
    parser_classes = [MultiPartParser]
    permission_classes = [
        permissions.IsAuthenticated,
        DownloadPushDeleteFileViewPermissions,
    ]

    def get(self, request, projectid, filename):
        key = utils.safe_join("projects/{}/meta/".format(projectid), filename)
        return utils2.storage.file_response(request, key, presigned=True)


class PublicFilesView(views.APIView):
    parser_classes = [MultiPartParser]
    permission_classes = []

    def get(self, request, filename):
        return utils2.storage.file_response(request, filename)


# docker-app/qfieldcloud/core/views/jobs_views.py

from django.core.exceptions import ObjectDoesNotExist
from qfieldcloud.core import permissions_utils, serializers
from qfieldcloud.core.models import Job, Project
from rest_framework import generics, permissions, viewsets
from rest_framework.response import Response
from rest_framework.status import HTTP_201_CREATED


class JobPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        project_id = permissions_utils.get_param_from_request(request, "project_id")

        try:
            project = Project.objects.get(id=project_id)
        except ObjectDoesNotExist:
            return False

        return permissions_utils.can_read_files(request.user, project)


class JobViewSet(viewsets.ReadOnlyModelViewSet):

    serializer_class = serializers.JobSerializer
    lookup_url_kwarg = "job_id"
    permission_classes = [permissions.IsAuthenticated]

    def get_serializer_by_job_type(self, job_type, *args, **kwargs):
        if job_type == Job.Type.DELTA_APPLY:
            return serializers.ApplyJobSerializer(*args, **kwargs)
        elif job_type == Job.Type.PACKAGE:
            return serializers.PackageJobSerializer(*args, **kwargs)
        elif job_type == Job.Type.PROCESS_PROJECTFILE:
            return serializers.ProcessProjectfileJobSerializer(*args, **kwargs)
        else:
            raise NotImplementedError(f'Unknown job type "{job_type}"')

    def get_serializer(self, *args, **kwargs):
        kwargs.setdefault("context", self.get_serializer_context())

        if self.action in ("create"):
            if "data" in kwargs:
                job_type = kwargs["data"]["type"]
            else:
                job_type = args[0].type

            return self.get_serializer_by_job_type(job_type, *args, **kwargs)

        if self.action in ("retrieve",):
            job_type = args[0].type

            return self.get_serializer_by_job_type(job_type, *args, **kwargs)

        return serializers.JobSerializer(*args, **kwargs)

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)

        if bool(int(request.data.get("force", 0))):
            serializer.is_valid(raise_exception=True)
            serializer.save()
        else:
            serializer.is_valid(raise_exception=True)
            if not serializer.Meta.allow_parallel_jobs:
                job = serializer.get_lastest_not_finished_job()
                if job:
                    return Response(self.get_serializer(job).data)
            serializer.save()

        return Response(serializer.data, status=HTTP_201_CREATED)

    def get_queryset(self):
        qs = Job.objects.select_subclasses()

        if self.action == "list":
            project_id = self.request.data.get("project_id")
            project = generics.get_object_or_404(Project, pk=project_id)
            qs = qs.filter(project=project)

        return qs


# docker-app/qfieldcloud/core/views/members_views.py

from django.contrib.auth import get_user_model
from django.core.exceptions import ObjectDoesNotExist
from django.utils.decorators import method_decorator
from drf_yasg.utils import swagger_auto_schema
from qfieldcloud.core import permissions_utils
from qfieldcloud.core.models import Organization, OrganizationMember
from qfieldcloud.core.serializers import OrganizationMemberSerializer
from rest_framework import generics, permissions, status
from rest_framework.response import Response
from rest_framework.settings import api_settings

User = get_user_model()


class ListCreateMembersViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        user = request.user
        organization_name = permissions_utils.get_param_from_request(
            request, "organization"
        )

        try:
            organization = User.objects.get(username=organization_name)
        except ObjectDoesNotExist:
            return False

        if request.method == "GET":
            return permissions_utils.can_read_members(user, organization)
        if request.method == "POST":
            return permissions_utils.can_create_members(user, organization)
        return False


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="List members of an organization",
        operation_id="List memebers",
    ),
)
@method_decorator(
    name="post",
    decorator=swagger_auto_schema(
        operation_description="Add a user as member of an organization",
        operation_id="Create member",
    ),
)
class ListCreateMembersView(generics.ListCreateAPIView):

    permission_classes = [permissions.IsAuthenticated, ListCreateMembersViewPermissions]
    serializer_class = OrganizationMemberSerializer

    def get_queryset(self):
        organization = self.request.parser_context["kwargs"]["organization"]
        organization_obj = User.objects.get(username=organization)

        return OrganizationMember.objects.filter(organization=organization_obj)

    def post(self, request, organization):

        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)

        organization_obj = Organization.objects.get(username=organization)
        member_obj = User.objects.get(username=request.data["member"])
        serializer.save(member=member_obj, organization=organization_obj)

        try:
            headers = {"Location": str(serializer.data[api_settings.URL_FIELD_NAME])}
        except (TypeError, KeyError):
            headers = {}

        return Response(
            serializer.data, status=status.HTTP_201_CREATED, headers=headers
        )


class GetUpdateDestroyMemberViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        user = request.user
        organization_name = permissions_utils.get_param_from_request(
            request, "organization"
        )

        try:
            organization = Organization.objects.get(username=organization_name)
        except ObjectDoesNotExist:
            return False

        if request.method == "GET":
            return permissions_utils.can_read_members(user, organization)
        if request.method in ["PUT", "PATCH"]:
            return permissions_utils.can_update_members(user, organization)
        if request.method in ["DELETE"]:
            return permissions_utils.can_delete_members(user, organization)
        return False


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="Get the role of a member of an organization",
        operation_id="Get memeber",
    ),
)
@method_decorator(
    name="put",
    decorator=swagger_auto_schema(
        operation_description="Update a memeber of an organization",
        operation_id="Update member",
    ),
)
@method_decorator(
    name="patch",
    decorator=swagger_auto_schema(
        operation_description="Partial update a member of an organization",
        operation_id="Patch member",
    ),
)
@method_decorator(
    name="delete",
    decorator=swagger_auto_schema(
        operation_description="Remove a member from an organization",
        operation_id="Delete member",
    ),
)
class GetUpdateDestroyMemberView(generics.RetrieveUpdateDestroyAPIView):

    permission_classes = [
        permissions.IsAuthenticated,
        GetUpdateDestroyMemberViewPermissions,
    ]
    serializer_class = OrganizationMemberSerializer

    def get_object(self):
        organization = self.request.parser_context["kwargs"]["organization"]
        member = self.request.parser_context["kwargs"]["username"]

        organization_obj = Organization.objects.get(username=organization)
        member_obj = User.objects.get(username=member)
        return OrganizationMember.objects.get(
            organization=organization_obj, member=member_obj
        )


# docker-app/qfieldcloud/core/views/package_views.py

from django.core.exceptions import ObjectDoesNotExist
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core import exceptions, permissions_utils, utils
from qfieldcloud.core.models import PackageJob, Project
from qfieldcloud.core.utils import (
    check_s3_key,
    get_project_files,
    get_project_package_files,
)
from qfieldcloud.core.utils2 import storage
from rest_framework import permissions, views
from rest_framework.response import Response


class PackageViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        try:
            project_id = request.parser_context["kwargs"].get("project_id")
            project = Project.objects.get(pk=project_id)
            return permissions_utils.can_read_project(request.user, project)
        except ObjectDoesNotExist:
            return False


class PackageUploadViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        if request.auth.client_type != AuthToken.ClientType.WORKER:
            return False

        try:
            project_id = request.parser_context["kwargs"].get("project_id")
            job_id = request.parser_context["kwargs"].get("job_id")
            project = Project.objects.get(pk=project_id)
            if not permissions_utils.can_update_project(request.user, project):
                return False

            # Check if the package job exists and it is already started, but not finished yet.
            # This is extra check that the request is coming from a currently active job.
            PackageJob.objects.get(
                id=job_id,
                status=PackageJob.Status.STARTED,
                project=project,
            )

            return True
        except ObjectDoesNotExist:
            return False


class LatestPackageView(views.APIView):

    permission_classes = [permissions.IsAuthenticated, PackageViewPermissions]

    def get(self, request, project_id):
        """Get last project package status and file list."""
        project = Project.objects.get(id=project_id)

        # Check if the project was packaged at least once
        if not project.last_package_job_id:
            raise exceptions.InvalidJobError(
                "Packaging has never been triggered or successful for this project."
            )

        filenames = set()
        files = []

        for f in get_project_package_files(project_id, project.last_package_job_id):
            filenames.add(f.name)
            files.append(
                {
                    "name": f.name,
                    "size": f.size,
                    "last_modified": f.last_modified,
                    "sha256": check_s3_key(f.key),
                    "md5sum": f.md5sum,
                    "is_attachment": False,
                }
            )

        # get attachment files directly from the original project files, not from the package
        for attachment_dir in project.attachment_dirs:
            for f in get_project_files(project_id, attachment_dir):
                # skip files that are part of the package
                if f.name in filenames:
                    continue

                filenames.add(f.name)
                files.append(
                    {
                        "name": f.name,
                        "size": f.size,
                        "last_modified": f.last_modified,
                        "sha256": check_s3_key(f.key),
                        "md5sum": f.md5sum,
                        "is_attachment": True,
                    }
                )

        if not files:
            raise exceptions.InvalidJobError("Empty project package.")

        last_job = project.last_package_job
        if last_job.feedback.get("feedback_version") == "2.0":
            layers = last_job.feedback["outputs"]["qgis_layers_data"]["layers_by_id"]
        else:
            steps = last_job.feedback.get("steps", [])
            layers = (
                steps[1]["outputs"]["layer_checks"]
                if len(steps) > 2 and steps[1].get("stage", 1) == 2
                else None
            )

        return Response(
            {
                "files": files,
                "layers": layers,
                "status": last_job.status,
                "package_id": last_job.pk,
                "packaged_at": last_job.project.data_last_packaged_at,
                "data_last_updated_at": last_job.project.data_last_updated_at,
            }
        )


class LatestPackageDownloadFilesView(views.APIView):

    permission_classes = [permissions.IsAuthenticated, PackageViewPermissions]

    def get(self, request, project_id, filename):
        """Download package file.

        Raises:
            exceptions.InvalidJobError: [description]
        """
        project = Project.objects.get(id=project_id)

        # Check if the project was packaged at least once
        if not project.last_package_job_id:
            raise exceptions.InvalidJobError(
                "Packaging has never been triggered or successful for this project."
            )

        key = f"projects/{project_id}/packages/{project.last_package_job_id}/{filename}"

        # files within attachment dirs that do not exist is the packaged files should be served
        # directly from the original data storage
        if storage.get_attachment_dir_prefix(project, filename) and not check_s3_key(
            key
        ):
            key = f"projects/{project_id}/files/{filename}"

        return storage.file_response(
            request, key, presigned=True, expires=600, as_attachment=True
        )


class PackageUploadFilesView(views.APIView):
    permission_classes = [permissions.IsAuthenticated, PackageUploadViewPermissions]

    def post(self, request, project_id, job_id, filename):
        """Upload the package files."""
        key = utils.safe_join(f"projects/{project_id}/packages/{job_id}/", filename)

        request_file = request.FILES.get("file")
        sha256sum = utils.get_sha256(request_file)
        md5sum = utils.get_md5sum(request_file)
        metadata = {"Sha256sum": sha256sum}

        bucket = utils.get_s3_bucket()
        bucket.upload_fileobj(request_file, key, ExtraArgs={"Metadata": metadata})

        return Response(
            {
                "name": filename,
                "size": request_file.size,
                "sha256": sha256sum,
                "md5sum": md5sum,
            }
        )


# docker-app/qfieldcloud/core/views/projects_views.py

from django.contrib.auth import get_user_model
from django.db import transaction
from django.utils.decorators import method_decorator
from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema
from qfieldcloud.core import exceptions, permissions_utils, utils
from qfieldcloud.core.models import Project, ProjectQueryset
from qfieldcloud.core.serializers import ProjectSerializer
from rest_framework import generics, permissions, viewsets

User = get_user_model()


class ProjectViewSetPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        if view.action == "list":
            # The queryset is already filtered by what the user can see
            return True
        user = request.user
        owner = permissions_utils.get_param_from_request(request, "owner")
        if owner:
            owner_obj = User.objects.get(username=owner)
        else:
            # If the owner is not in the request, means that the owner
            # should be the user that made the request
            owner_obj = user

        if view.action == "create":
            return permissions_utils.can_create_project(user, owner_obj)

        projectid = permissions_utils.get_param_from_request(request, "projectid")
        project = Project.objects.get(id=projectid)

        if view.action == "retrieve":
            return permissions_utils.can_read_project(user, project)
        elif view.action == "destroy":
            return permissions_utils.can_delete_project(user, project)
        elif view.action in ["update", "partial_update"]:
            return permissions_utils.can_update_project(user, project)

        return False


include_public_param = openapi.Parameter(
    "include-public",
    openapi.IN_QUERY,
    description="Include public projects",
    type=openapi.TYPE_BOOLEAN,
)


@method_decorator(
    name="retrieve",
    decorator=swagger_auto_schema(
        operation_description="Get a project",
        operation_id="Get a project",
    ),
)
@method_decorator(
    name="update",
    decorator=swagger_auto_schema(
        operation_description="Update a project",
        operation_id="Update a project",
    ),
)
@method_decorator(
    name="partial_update",
    decorator=swagger_auto_schema(
        operation_description="Patch a project",
        operation_id="Patch a project",
    ),
)
@method_decorator(
    name="destroy",
    decorator=swagger_auto_schema(
        operation_description="Delete a project",
        operation_id="Delete a project",
    ),
)
@method_decorator(
    name="list",
    decorator=swagger_auto_schema(
        operation_description="""List projects owned by the authenticated
        user or that the authenticated user has explicit permission to access
        (i.e. she is a project collaborator)""",
        operation_id="List projects",
        manual_parameters=[include_public_param],
    ),
)
@method_decorator(
    name="create",
    decorator=swagger_auto_schema(
        operation_description="""Create a new project owned by the specified
        user or organization""",
        operation_id="Create a project",
    ),
)
class ProjectViewSet(viewsets.ModelViewSet):

    serializer_class = ProjectSerializer
    lookup_url_kwarg = "projectid"
    permission_classes = [permissions.IsAuthenticated, ProjectViewSetPermissions]

    def get_queryset(self):

        projects = Project.objects.for_user(self.request.user)

        # In the list endpoint, by default we filter out public projects. They can be
        # included with the `include-public` query parameter.
        if self.action == "list":
            include_public = False
            include_public_param = self.request.query_params.get("include-public")
            if include_public_param and include_public_param.lower() == "1":
                include_public = True

            if not include_public:
                projects = projects.exclude(
                    user_role_origin=ProjectQueryset.RoleOrigins.PUBLIC
                )

        return projects

    @transaction.atomic
    def perform_update(self, serializer):
        # Here we do an additional check if the owner has changed. If so, the reciever
        # of the project must have enough storage quota, otherwise the transfer is
        # not permitted.

        # TODO: this should be moved to some more reusable place. Maybe in Project.clean() ?
        # But then it would also be enforced by admin, which we don't want I guess...

        old_owner = serializer.instance.owner
        super().perform_update(serializer)
        new_owner = serializer.instance.owner

        # If owner has not changed, no additional check is made
        if old_owner == new_owner:
            return

        # Owner has changed, we must ensure he has enough quota for that
        # (in this transaction, the project is his already, so we just need to
        # check his quota)
        if new_owner.useraccount.storage_quota_left_mb < 0:
            # If not, we rollback the transaction
            # (don't give away numbers in message as it's potentially private)
            raise exceptions.QuotaError(
                "Project storage too large for recipient's quota."
            )

    def destroy(self, request, projectid):
        # Delete files from storage
        bucket = utils.get_s3_bucket()
        prefix = utils.safe_join("projects/{}/".format(projectid))
        bucket.objects.filter(Prefix=prefix).delete()

        return super().destroy(request, projectid)


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="List public projects",
        operation_id="List public projects",
    ),
)
class PublicProjectsListView(generics.ListAPIView):

    permission_classes = [permissions.IsAuthenticated]
    serializer_class = ProjectSerializer

    def get_queryset(self):
        return Project.objects.for_user(self.request.user).filter(is_public=True)


# docker-app/qfieldcloud/core/views/qfield_files_views.py

from pathlib import PurePath

from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from django.http.response import HttpResponseRedirect
from django.utils.decorators import method_decorator
from drf_yasg.utils import swagger_auto_schema
from qfieldcloud.core import exceptions, permissions_utils, serializers, utils
from qfieldcloud.core.models import PackageJob, Project
from rest_framework import permissions, views
from rest_framework.response import Response


class PackageViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        projectid = permissions_utils.get_param_from_request(request, "projectid")
        try:
            project = Project.objects.get(id=projectid)
        except ObjectDoesNotExist:
            return False
        user = request.user
        return permissions_utils.can_read_files(user, project)


@method_decorator(
    name="post",
    decorator=swagger_auto_schema(
        operation_description="Launch QField packaging project",
        operation_id="Launch qfield packaging",
    ),
)
@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="Get QField packaging status",
        operation_id="Get qfield packaging status",
    ),
)
class PackageView(views.APIView):

    permission_classes = [permissions.IsAuthenticated, PackageViewPermissions]

    def post(self, request, projectid):

        project_obj = Project.objects.get(id=projectid)

        if not project_obj.project_filename:
            raise exceptions.NoQGISProjectError()

        # Check if active packaging job already exists
        # TODO: !!!!!!!!!!!! cache results for some minutes
        query = Q(project=project_obj) & (
            Q(status=PackageJob.Status.PENDING)
            | Q(status=PackageJob.Status.QUEUED)
            | Q(status=PackageJob.Status.STARTED)
        )

        # NOTE uncomment to enforce job creation
        # PackageJob.objects.filter(query).delete()

        if not project_obj.needs_repackaging:
            export_job = (
                PackageJob.objects.filter(status=PackageJob.Status.FINISHED)
                .filter(project=project_obj)
                .latest("started_at")
            )
            if export_job:
                serializer = serializers.ExportJobSerializer(export_job)
                return Response(serializer.data)

        if PackageJob.objects.filter(query).exists():
            serializer = serializers.ExportJobSerializer(PackageJob.objects.get(query))
            return Response(serializer.data)

        export_job = PackageJob.objects.create(
            project=project_obj, created_by=self.request.user
        )

        # TODO: check if user is allowed otherwise ERROR 403
        serializer = serializers.ExportJobSerializer(export_job)

        return Response(serializer.data)

    def get(self, request, projectid):
        project_obj = Project.objects.get(id=projectid)

        export_job = (
            PackageJob.objects.filter(project=project_obj).order_by("updated_at").last()
        )

        serializer = serializers.ExportJobSerializer(export_job)
        return Response(serializer.data)


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="List QField project files",
        operation_id="List qfield project files",
    ),
)
class ListFilesView(views.APIView):

    permission_classes = [permissions.IsAuthenticated, PackageViewPermissions]

    def get(self, request, projectid):

        project_obj = Project.objects.get(id=projectid)

        # Check if the project was exported at least once
        if not PackageJob.objects.filter(
            project=project_obj, status=PackageJob.Status.FINISHED
        ):
            raise exceptions.InvalidJobError(
                "Project files have not been exported for the provided project id"
            )

        package_job = project_obj.last_package_job
        assert package_job

        # Obtain the bucket object
        bucket = utils.get_s3_bucket()

        export_prefix = f"projects/{projectid}/packages/{package_job.id}/"

        files = []
        for obj in bucket.objects.filter(Prefix=export_prefix):
            path = PurePath(obj.key)

            # We cannot be sure of the metadata's first letter case
            # https://github.com/boto/boto3/issues/1709
            metadata = obj.Object().metadata
            if "sha256sum" in metadata:
                sha256sum = metadata["sha256sum"]
            else:
                sha256sum = metadata["Sha256sum"]

            files.append(
                {
                    # Get the path of the file relative to the export directory
                    "name": str(path.relative_to(*path.parts[:4])),
                    "size": obj.size,
                    "sha256": sha256sum,
                }
            )

        if package_job.feedback.get("feedback_version") == "2.0":
            layers = package_job.feedback["outputs"]["qgis_layers_data"]["layers_by_id"]

            for data in layers.values():
                data["valid"] = data["is_valid"]
                data["status"] = data["error_code"]
        else:
            steps = package_job.feedback.get("steps", [])
            layers = (
                steps[1]["outputs"]["layer_checks"]
                if len(steps) > 2 and steps[1].get("stage", 1) == 2
                else None
            )

        return Response(
            {
                "files": files,
                "layers": layers,
                "exported_at": package_job.updated_at,
                "export_id": package_job.pk,
            }
        )


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="Download file for QField",
        operation_id="Download qfield file",
    ),
)
class DownloadFileView(views.APIView):

    permission_classes = [permissions.IsAuthenticated, PackageViewPermissions]

    def get(self, request, projectid, filename):

        project_obj = Project.objects.get(id=projectid)
        package_job = project_obj.last_package_job

        # Check if the project was exported at least once
        if not package_job:
            raise exceptions.InvalidJobError(
                "Project files have not been exported for the provided project id"
            )

        filekey = utils.safe_join(
            f"projects/{projectid}/packages/{package_job.id}/", filename
        )

        url = utils.get_s3_client().generate_presigned_url(
            "get_object",
            Params={
                "Key": filekey,
                "Bucket": utils.get_s3_bucket().name,
                "ResponseContentType": "application/force-download",
                "ResponseContentDisposition": f'attachment;filename="{filename}"',
            },
            ExpiresIn=60,
            HttpMethod="GET",
        )

        return HttpResponseRedirect(url)


# docker-app/qfieldcloud/core/views/status_views.py

from django.conf import settings
from django.core.cache import cache
from django.utils.decorators import method_decorator
from drf_yasg.utils import swagger_auto_schema
from qfieldcloud.core import geodb_utils, utils
from rest_framework import status, views
from rest_framework.permissions import AllowAny
from rest_framework.response import Response


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="Get the current status of the APIs",
        operation_id="Get status",
    ),
)
class APIStatusView(views.APIView):
    permission_classes = [AllowAny]

    def get(self, request):
        # Try to get the status from the cache
        results = cache.get("status_results", {})
        if not results:

            results["redis"] = "ok"
            # Check if redis is visible
            if not utils.redis_is_running():
                results["redis"] = "error"

            results["geodb"] = "ok"
            # Check geodb
            if not geodb_utils.geodb_is_running():
                results["geodb"] = "error"

            results["storage"] = "ok"
            # Check if bucket exists (i.e. the connection works)
            try:
                s3_client = utils.get_s3_client()
                s3_client.head_bucket(Bucket=settings.STORAGE_BUCKET_NAME)
            except Exception:
                results["storage"] = "error"

            # Cache the result for 10 minutes
            cache.set("status_results", results, 600)

        return Response(results, status=status.HTTP_200_OK)


# docker-app/qfieldcloud/core/views/users_views.py

from django.contrib.auth import get_user_model
from django.core.exceptions import ObjectDoesNotExist
from django.utils.decorators import method_decorator
from drf_yasg.utils import swagger_auto_schema
from qfieldcloud.core import permissions_utils, querysets_utils
from qfieldcloud.core.models import Organization, Project
from qfieldcloud.core.serializers import (
    CompleteUserSerializer,
    OrganizationSerializer,
    PublicInfoUserSerializer,
)
from rest_framework import generics, permissions
from rest_framework.response import Response

User = get_user_model()


class ListUsersViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        return permissions_utils.can_list_users_organizations(request.user)


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="List all users and organizations",
        operation_id="List users and organizations",
    ),
)
class ListUsersView(generics.ListAPIView):

    serializer_class = PublicInfoUserSerializer
    permission_classes = [permissions.IsAuthenticated, ListUsersViewPermissions]
    paginate_by = 100

    def get_queryset(self):
        params = self.request.GET
        query = params.get("q", "")

        project = None
        if params.get("project"):
            try:
                project = Project.objects.get(id=params.get("project"))
            except Project.DoesNotExist:
                pass

        organization = None
        if params.get("organization"):
            try:
                organization = Organization.objects.get(
                    username=params.get("organization")
                )
            except Project.DoesNotExist:
                pass

        # TODO : are these GET paremters documented somewhere ? Shouldn't we use something
        # like django_filters.rest_framework.DjangoFilterBackend so they get auto-documented
        # in DRF's views, or is that supposedly done with swagger ?
        exclude_organizations = bool(int(params.get("exclude_organizations") or 0))
        exclude_teams = bool(int(params.get("exclude_teams") or 0))
        invert = bool(int(params.get("invert") or 0))
        return querysets_utils.get_users(
            query,
            project=project,
            organization=organization,
            exclude_organizations=exclude_organizations,
            exclude_teams=exclude_teams,
            invert=invert,
        )


class RetrieveUpdateUserViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):

        username = permissions_utils.get_param_from_request(request, "username")

        try:
            user = User.objects.get(username=username)
        except ObjectDoesNotExist:
            return False

        if request.method == "GET":
            # The queryset is already filtered by what the user can see
            return True
        if request.method in ["PUT", "PATCH"]:
            return permissions_utils.can_update_user(request.user, user)
        return False


@method_decorator(
    name="get",
    decorator=swagger_auto_schema(
        operation_description="""Get a single user's (or organization) publicly
        information or complete info if the request is done by the user
        himself""",
        operation_id="Get user",
    ),
)
@method_decorator(
    name="put",
    decorator=swagger_auto_schema(
        operation_description="Update a user",
        operation_id="Update a user",
    ),
)
@method_decorator(
    name="patch",
    decorator=swagger_auto_schema(
        operation_description="Patch a user",
        operation_id="Patch a user",
    ),
)
class RetrieveUpdateUserView(generics.RetrieveUpdateAPIView):
    """Get or Update the authenticated user"""

    permission_classes = [
        permissions.IsAuthenticated,
        RetrieveUpdateUserViewPermissions,
    ]
    serializer_class = CompleteUserSerializer

    def get_object(self):
        username = self.request.parser_context["kwargs"]["username"]
        return User.objects.get(username=username)

    def get(self, request, username):

        user = User.objects.get(username=username)

        if user.user_type == User.TYPE_ORGANIZATION:
            organization = Organization.objects.of_user(request.user).get(
                username=username
            )
            serializer = OrganizationSerializer(organization)
        else:
            if request.user == user:
                serializer = CompleteUserSerializer(user)
            else:
                serializer = PublicInfoUserSerializer(user)

        return Response(serializer.data)


class ListUserOrganizationsViewPermissions(permissions.BasePermission):
    def has_permission(self, request, view):
        username = permissions_utils.get_param_from_request(request, "username")

        if request.user.username != username:
            return False

        if request.method == "GET":
            return True

        return False


class ListUserOrganizationsView(generics.ListAPIView):
    """Get user's organizations"""

    permission_classes = [
        permissions.IsAuthenticated,
        ListUserOrganizationsViewPermissions,
    ]
    serializer_class = OrganizationSerializer

    def get_queryset(self):
        return Organization.objects.of_user(self.request.user)


# docker-app/qfieldcloud/settings.py

"""
Django settings for qfieldcloud project.

Generated by 'django-admin startproject' using Django 2.2.6.

For more information on this file, see
https://docs.djangoproject.com/en/2.2/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/2.2/ref/settings/
"""

import os
from datetime import timedelta

import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get("SECRET_KEY")

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = int(os.environ.get("DEBUG", default=0))

# if we are in debug, we need to update the internal IPS to make the
# debug toolbar work within docker
if DEBUG:
    import socket  # only if you haven't already imported this

    hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + [
        "127.0.0.1",
        "10.0.2.2",
    ]

ENVIRONMENT = os.environ.get("ENVIRONMENT")

# 'DJANGO_ALLOWED_HOSTS' should be a single string of hosts with a space between each.
# For example: 'DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1]'
ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS").split(" ")

AUTHENTICATION_BACKENDS = [
    "axes.backends.AxesBackend",
    # Needed to login by username in Django admin, regardless of `allauth`
    "django.contrib.auth.backends.ModelBackend",
    # `allauth` specific authentication methods, such as login by email
    "allauth.account.auth_backends.AuthenticationBackend",
]


# Application definition
INSTALLED_APPS = [
    # django contrib
    "django.contrib.admin",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.gis",
    "django.contrib.sessions",
    "django.contrib.messages",
    "django.contrib.staticfiles",
    "django.contrib.sites",
    # 3rd-party apps
    # if django_filters defined after [rest_framework] caused '... _frozen_importlib._DeadlockError ...'
    # https://stackoverflow.com/questions/55844680/deadlock-detected-when-trying-to-start-server
    "django_tables2",
    "django_filters",
    # debug
    "debug_toolbar",
    # style
    "bootstrap4",
    "rest_framework",
    "rest_framework.authtoken",
    "drf_yasg",
    "allauth",
    "allauth.account",
    "allauth.socialaccount",
    "storages",  # Integration with S3 Storages
    "invitations",
    "django_cron",
    "timezone_field",
    "auditlog",
    # Local
    "qfieldcloud.core",
    "qfieldcloud.subscription",
    "qfieldcloud.notifs",
    "qfieldcloud.authentication",
    # 3rd party - keep at bottom to allow overrides
    "notifications",
    "axes",
    "migrate_sql",
]

MIDDLEWARE = [
    "debug_toolbar.middleware.DebugToolbarMiddleware",
    "django.middleware.security.SecurityMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.locale.LocaleMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django.contrib.messages.middleware.MessageMiddleware",
    "django.middleware.clickjacking.XFrameOptionsMiddleware",
    "django_currentuser.middleware.ThreadLocalUserMiddleware",
    "auditlog.middleware.AuditlogMiddleware",
    "qfieldcloud.core.middleware.timezone.TimezoneMiddleware",
    "qfieldcloud.core.middleware.test.TestMiddleware",
    "axes.middleware.AxesMiddleware",
]

CRON_CLASSES = [
    "qfieldcloud.notifs.cron.SendNotificationsJob",
    # "qfieldcloud.core.cron.DeleteExpiredInvitationsJob",
    "qfieldcloud.core.cron.ResendFailedInvitationsJob",
    "qfieldcloud.core.cron.SetTerminatedWorkersToFinalStatusJob",
    "qfieldcloud.core.cron.DeleteObsoleteProjectPackagesJob",
]

ROOT_URLCONF = "qfieldcloud.urls"

TEMPLATES = [
    {
        "BACKEND": "django.template.backends.django.DjangoTemplates",
        "DIRS": [
            os.path.join(BASE_DIR, "qfieldcloud", "core", "templates"),
        ],
        "APP_DIRS": True,
        "OPTIONS": {
            "builtins": [],
            "context_processors": [
                "django.template.context_processors.debug",
                "django.template.context_processors.request",
                "django.contrib.auth.context_processors.auth",
                "django.contrib.messages.context_processors.messages",
            ],
        },
    },
]

WSGI_APPLICATION = "qfieldcloud.wsgi.application"


# Database
# https://docs.djangoproject.com/en/2.2/ref/settings/#databases

DATABASES = {
    "default": {
        "ENGINE": "django.contrib.gis.db.backends.postgis",
        "NAME": os.environ.get("SQL_DATABASE"),
        "USER": os.environ.get("SQL_USER"),
        "PASSWORD": os.environ.get("SQL_PASSWORD"),
        "HOST": os.environ.get("SQL_HOST"),
        "PORT": os.environ.get("SQL_PORT"),
        "OPTIONS": {"sslmode": os.environ.get("SQL_SSLMODE")},
        "TEST": {
            "NAME": os.environ.get("SQL_DATABASE_TEST"),
        },
    }
}

# Password validation
# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
    },
    {
        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
    },
    {
        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
    },
    {
        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
    },
]


# Internationalization
# https://docs.djangoproject.com/en/2.2/topics/i18n/

LANGUAGE_CODE = "en-us"

TIME_ZONE = "Europe/Zurich"

USE_I18N = True

USE_L10N = True

USE_TZ = True


# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.2/howto/static-files/

STATIC_URL = "/staticfiles/"
STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles")
STATICFILES_DIRS = []
STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"


MEDIA_URL = "/mediafiles/"
MEDIA_ROOT = os.path.join(BASE_DIR, "mediafiles")

# S3 Storage
STORAGE_ACCESS_KEY_ID = os.environ.get("STORAGE_ACCESS_KEY_ID")
STORAGE_SECRET_ACCESS_KEY = os.environ.get("STORAGE_SECRET_ACCESS_KEY")
STORAGE_BUCKET_NAME = os.environ.get("STORAGE_BUCKET_NAME")
STORAGE_REGION_NAME = os.environ.get("STORAGE_REGION_NAME")
STORAGE_ENDPOINT_URL = os.environ.get("STORAGE_ENDPOINT_URL")

AUTH_USER_MODEL = "core.User"

# QFieldCloud variables
AUTH_TOKEN_LENGTH = 100
AUTH_TOKEN_EXPIRATION_HOURS = 24 * 30

REST_FRAMEWORK = {
    "DEFAULT_PERMISSION_CLASSES": [
        "rest_framework.permissions.IsAuthenticated",
    ],
    "DEFAULT_AUTHENTICATION_CLASSES": [
        "qfieldcloud.authentication.authentication.TokenAuthentication",
        "rest_framework.authentication.SessionAuthentication",
    ],
    "DEFAULT_SCHEMA_CLASS": "rest_framework.schemas.coreapi.AutoSchema",
    "EXCEPTION_HANDLER": "qfieldcloud.core.rest_utils.exception_handler",
}

EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"

SITE_ID = 1

SWAGGER_SETTINGS = {
    "LOGIN_URL": "rest_framework:login",
    "LOGOUT_URL": "rest_framework:logout",
}

LOGIN_URL = "account_login"


sentry_sdk.init(
    dsn=os.environ.get("SENTRY_DSN", ""),
    integrations=[DjangoIntegration()],
    # Define how many random events are sent for performance monitoring
    sample_rate=0.05,
    server_name=os.environ.get("QFIELDCLOUD_HOST"),
    # If you wish to associate users to errors (assuming you are using
    # django.contrib.auth) you may enable sending PII data.
    send_default_pii=True,
)


# Django allauth configurations
# https://django-allauth.readthedocs.io/en/latest/configuration.html
ACCOUNT_AUTHENTICATION_METHOD = "username_email"
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = 3
ACCOUNT_EMAIL_SUBJECT_PREFIX = ""

# Choose one of "mandatory", "optional", or "none".
# For local development and test use "optional" or "none"
ACCOUNT_EMAIL_VERIFICATION = os.environ.get("ACCOUNT_EMAIL_VERIFICATION")
ACCOUNT_PRESERVE_USERNAME_CASING = False
ACCOUNT_USERNAME_REQUIRED = True
ACCOUNT_ADAPTER = "invitations.models.InvitationsAdapter"
ACCOUNT_LOGOUT_ON_GET = True

# Django axes configuration
# https://django-axes.readthedocs.io/en/latest/4_configuration.html
###########################
# The integer number of login attempts allowed before a record is created for the failed logins. Default: 3
AXES_FAILURE_LIMIT = 5
# If True, only lock based on username, and never lock based on IP if attempts exceed the limit. Otherwise utilize the existing IP and user locking logic. Default: False
AXES_ONLY_USER_FAILURES = True
# If set, defines a period of inactivity after which old failed login attempts will be cleared. If an integer, will be interpreted as a number of hours. Default: None
AXES_COOLOFF_TIME = timedelta(minutes=30)
# If True, a successful login will reset the number of failed logins. Default: False
AXES_RESET_ON_SUCCESS = True

# Django email configuration
EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend"
EMAIL_HOST = os.environ.get("EMAIL_HOST")
EMAIL_USE_TLS = os.environ.get("EMAIL_USE_TLS").lower() == "true"
EMAIL_USE_SSL = os.environ.get("EMAIL_USE_SSL").lower() == "true"
EMAIL_PORT = os.environ.get("EMAIL_PORT")
EMAIL_HOST_USER = os.environ.get("EMAIL_HOST_USER")
EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_HOST_PASSWORD")
DEFAULT_FROM_EMAIL = os.environ.get("DEFAULT_FROM_EMAIL")


# Django invitations configurations
# https://github.com/bee-keeper/django-invitations#additional-configuration
INVITATIONS_INVITATION_EXPIRY = 365  # integer in days, 0 disables invitations
INVITATIONS_INVITATION_ONLY = False
INVITATIONS_ACCEPT_INVITE_AFTER_SIGNUP = True
INVITATIONS_GONE_ON_ACCEPT_ERROR = False

TEST_RUNNER = "qfieldcloud.testing.QfcTestSuiteRunner"

LOGLEVEL = os.environ.get("LOGLEVEL", "DEBUG").upper()
LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "formatters": {
        "json": {
            "()": "qfieldcloud.core.logging.formatters.CustomisedJSONFormatter",
        },
    },
    "handlers": {
        "console.json": {
            "class": "logging.StreamHandler",
            "formatter": "json",
        },
    },
    "root": {
        "handlers": ["console.json"],
        "level": "INFO",
    },
}

DEFAULT_AUTO_FIELD = "django.db.models.AutoField"

# Whether we are currently running tests
# NOTE automatically set when running tests, don't change manually!
IN_TEST_SUITE = False

QFIELDCLOUD_TOKEN_SERIALIZER = "qfieldcloud.core.serializers.TokenSerializer"
QFIELDCLOUD_USER_SERIALIZER = "qfieldcloud.core.serializers.CompleteUserSerializer"

WORKER_TIMEOUT_S = int(os.environ.get("QFIELDCLOUD_WORKER_TIMEOUT_S", 60))
APPLY_DELTAS_LIMIT = 1000

DEBUG_TOOLBAR_CONFIG = {
    "SHOW_TOOLBAR_CALLBACK": lambda r: DEBUG and ENVIRONMENT == "development",
}

QFIELDCLOUD_ADMIN_URI = os.environ.get("QFIELDCLOUD_ADMIN_URI", "admin/")


# docker-app/qfieldcloud/testing.py

from django.conf import settings
from django.test.runner import DiscoverRunner


class QfcTestSuiteRunner(DiscoverRunner):
    def __init__(self, *args, **kwargs):
        settings.IN_TEST_SUITE = True
        super().__init__(*args, **kwargs)


# docker-app/qfieldcloud/urls.py

"""qfieldcloud URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path(
        '', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path(
        '', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path(
        'blog/', include('blog.urls'))
"""
from django.conf import settings
from django.contrib import admin
from django.urls import include, path, re_path
from django.utils.translation import gettext as _
from drf_yasg import openapi
from drf_yasg.views import get_schema_view
from qfieldcloud.authentication import views as auth_views
from qfieldcloud.core.views import files_views
from rest_framework import permissions

admin.site.site_header = _("QFieldCloud Admin")
admin.site.site_title = _("QFieldCloud Admin")
admin.site.index_title = _("Welcome to QFieldCloud Admin")

schema_view = get_schema_view(
    openapi.Info(
        title="QFieldcloud REST API",
        default_version="v1",
        description="Test description",
        terms_of_service="https://",
        contact=openapi.Contact(email="info@opengis.ch"),
        license=openapi.License(name="License"),
    ),
    public=True,
    permission_classes=(permissions.AllowAny,),
)

urlpatterns = [
    re_path(
        r"^swagger(?P<format>\.json|\.yaml)$",
        schema_view.without_ui(cache_timeout=0),
        name="schema-json",
    ),
    path(
        "swagger/",
        schema_view.with_ui("swagger", cache_timeout=0),
        name="schema-swagger-ui",
    ),
    path(
        settings.QFIELDCLOUD_ADMIN_URI + "api/files/<uuid:projectid>/",
        files_views.ListFilesView.as_view(permission_classes=[permissions.IsAdminUser]),
    ),
    path(
        settings.QFIELDCLOUD_ADMIN_URI + "api/files/<uuid:projectid>/<path:filename>/",
        files_views.DownloadPushDeleteFileView.as_view(
            permission_classes=[permissions.IsAdminUser]
        ),
        name="project_file_download",
    ),
    path(settings.QFIELDCLOUD_ADMIN_URI, admin.site.urls),
    path("docs/", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"),
    path("api/v1/auth/login/", auth_views.LoginView.as_view()),
    path("api/v1/auth/token/", auth_views.LoginView.as_view()),
    path("api/v1/auth/user/", auth_views.UserView.as_view()),
    path("api/v1/auth/logout/", auth_views.LogoutView.as_view()),
    path("api/v1/", include("qfieldcloud.core.urls")),
    path("auth/", include("rest_framework.urls")),
    path("accounts/", include("allauth.urls")),
    re_path(r"^invitations/", include("invitations.urls", namespace="invitations")),
    path("__debug__/", include("debug_toolbar.urls")),
]


# docker-app/qfieldcloud/wsgi.py

"""
WSGI config for qfieldcloud project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/
"""

import os

from django.core.wsgi import get_wsgi_application

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "qfieldcloud.settings")

application = get_wsgi_application()


# docker-app/qfieldcloud/core/admin.py

import json
import time
from typing import Any, Dict

from allauth.socialaccount.models import SocialAccount, SocialApp, SocialToken
from django.contrib import admin, messages
from django.contrib.admin.templatetags.admin_urls import admin_urlname
from django.contrib.auth.models import Group
from django.db.models.fields.json import JSONField
from django.forms import ModelForm, fields, widgets
from django.http.response import HttpResponseRedirect
from django.shortcuts import resolve_url
from django.utils.html import escape, format_html
from django.utils.safestring import SafeText
from qfieldcloud.core import exceptions
from qfieldcloud.core.models import (
    ApplyJob,
    ApplyJobDelta,
    Delta,
    Geodb,
    Organization,
    OrganizationMember,
    PackageJob,
    ProcessProjectfileJob,
    Project,
    ProjectCollaborator,
    Team,
    TeamMember,
    User,
    UserAccount,
)
from qfieldcloud.core.utils2 import jobs


class PrettyJSONWidget(widgets.Textarea):
    def format_value(self, value):
        text_value = json.dumps(json.loads(value), indent=2, sort_keys=True)

        row_lengths = [len(r) for r in text_value.split("\n")]

        self.attrs["rows"] = min(max(len(row_lengths) + 2, 10), 30)
        self.attrs["cols"] = min(max(max(row_lengths) + 2, 40), 120)
        return SafeText(text_value)


def search_parser(
    _request, _queryset, search_term: str, filter_config: Dict[str, Dict[str, str]]
) -> Dict[str, Any]:
    custom_filter = {}
    CUSTOM_SEARCH_DIVIDER = ":"
    if CUSTOM_SEARCH_DIVIDER in search_term:
        prefix, search = search_term.split(CUSTOM_SEARCH_DIVIDER, 1)
        prefix_config = filter_config.get(prefix)
        if prefix_config:
            extra_filters = prefix_config.get("extra_filters", {})
            filter_keyword = prefix_config["filter"]

            custom_filter = {**custom_filter, **extra_filters}
            custom_filter[filter_keyword] = search

    return custom_filter


def model_admin_url(obj, name: str = None) -> str:
    url = resolve_url(admin_urlname(obj._meta, SafeText("change")), obj.pk)
    return format_html('<a href="{}">{}</a>', url, name or str(obj))


def format_pre(value):
    return format_html("<pre>{}</pre>", escape(value))


def format_pre_json(value):
    if value:
        text_value = json.dumps(value, indent=2, sort_keys=True)
        return format_pre(text_value)
    else:
        return format_pre(value)


class GeodbInline(admin.TabularInline):
    model = Geodb
    extra = 0

    def has_add_permission(self, request, obj):
        return False

    def has_delete_permission(self, request, obj):
        return False

    def has_change_permission(self, request, obj):
        return False


class MemberOrganizationInline(admin.TabularInline):
    model = OrganizationMember
    extra = 0

    def has_add_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type in (User.TYPE_USER, User.TYPE_ORGANIZATION)

    def has_delete_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type in (User.TYPE_USER, User.TYPE_ORGANIZATION)

    def has_change_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type in (User.TYPE_USER, User.TYPE_ORGANIZATION)


class MemberTeamInline(admin.TabularInline):
    model = TeamMember
    extra = 0

    def has_add_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type in (User.TYPE_USER, User.TYPE_ORGANIZATION)

    def has_delete_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type in (User.TYPE_USER, User.TYPE_ORGANIZATION)

    def has_change_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type in (User.TYPE_USER, User.TYPE_ORGANIZATION)


class UserAccountInline(admin.StackedInline):
    model = UserAccount
    extra = 1

    def has_add_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type in (User.TYPE_USER, User.TYPE_ORGANIZATION)

    def has_delete_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type in (User.TYPE_USER, User.TYPE_ORGANIZATION)


class ProjectInline(admin.TabularInline):
    model = Project
    extra = 0

    fields = ("owned_project", "is_public", "overwrite_conflicts")
    readonly_fields = ("owned_project",)

    def owned_project(self, obj):
        return model_admin_url(obj, obj.name)

    def has_add_permission(self, request, obj):
        return False

    def has_delete_permission(self, request, obj):
        return False

    def has_change_permission(self, request, obj):
        return False


class UserProjectCollaboratorInline(admin.TabularInline):
    model = ProjectCollaborator
    extra = 0

    def has_add_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type == User.TYPE_USER

    def has_delete_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type == User.TYPE_USER

    def has_change_permission(self, request, obj):
        if obj is None:
            return True
        return obj.user_type == User.TYPE_USER


class UserAdmin(admin.ModelAdmin):
    list_display = (
        "username",
        "first_name",
        "last_name",
        "email",
        "is_superuser",
        "is_staff",
        "is_active",
        "date_joined",
        "user_type",
        "useraccount",
    )
    list_filter = (
        "user_type",
        "date_joined",
        "is_active",
        "is_staff",
        "useraccount__plan",
    )

    search_fields = ("username__icontains", "owner__username__iexact")

    fields = (
        "username",
        "password",
        "email",
        "first_name",
        "last_name",
        "remaining_invitations",
        "date_joined",
        "last_login",
        "is_superuser",
        "is_staff",
        "is_active",
        "has_newsletter_subscription",
        "has_accepted_tos",
    )

    inlines = (
        UserAccountInline,
        GeodbInline,
    )

    change_form_template = "admin/user_change_form.html"

    def save_model(self, request, obj, form, change):
        # Set the password to the value in the field if it's changed.
        if obj.pk:
            if "password" in form.changed_data:
                obj.set_password(obj.password)
        else:
            obj.set_password(obj.password)
        obj.save()


class ProjectCollaboratorInline(admin.TabularInline):
    model = ProjectCollaborator
    extra = 0


class ProjectFilesWidget(widgets.Input):
    template_name = "admin/project_files_widget.html"


class ProjectForm(ModelForm):
    project_files = fields.CharField(
        disabled=True, required=False, widget=ProjectFilesWidget
    )

    class Meta:
        model = Project
        widgets = {"project_filename": widgets.TextInput()}
        fields = "__all__"  # required for Django 3.x


class ProjectAdmin(admin.ModelAdmin):
    form = ProjectForm
    list_display = (
        "id",
        "name",
        "owner",
        "is_public",
        "description",
        "created_at",
        "updated_at",
    )
    list_filter = (
        "is_public",
        "created_at",
        "updated_at",
    )
    fields = (
        "id",
        "name",
        "description",
        "is_public",
        "owner",
        "project_filename",
        "storage_size_mb",
        "created_at",
        "updated_at",
        "data_last_updated_at",
        "data_last_packaged_at",
        "project_details__pre",
        "project_files",
    )
    readonly_fields = (
        "id",
        "storage_size_mb",
        "created_at",
        "updated_at",
        "data_last_updated_at",
        "data_last_packaged_at",
        "project_details__pre",
    )
    inlines = (ProjectCollaboratorInline,)
    search_fields = (
        "id",
        "name__icontains",
        "owner__username__iexact",
    )

    def get_search_results(self, request, queryset, search_term):
        filters = search_parser(
            request,
            queryset,
            search_term,
            {
                "owner": {
                    "filter": "owner__username__iexact",
                },
                "collaborator": {
                    "filter": "user_roles__user__username__iexact",
                    "extra_filters": {
                        "is_public": False,
                    },
                },
            },
        )

        if filters:
            return queryset.filter(**filters), True

        queryset, use_distinct = super().get_search_results(
            request, queryset, search_term
        )

        return queryset, use_distinct

    def project_files(self, instance):
        return instance.pk

    def project_details__pre(self, instance):
        if instance.project_details is None:
            return ""

        return format_pre_json(instance.project_details)


class DeltaInline(admin.TabularInline):
    model = ApplyJob.deltas_to_apply.through

    fields = (
        "delta",
        "status",
        # TODO find a way to use dynamic fields
        # "feedback__pre",
    )

    def has_add_permission(self, request, obj):
        return False

    def has_delete_permission(self, request, obj):
        return False

    # def feedback__pre(self, instance):
    #     return format_pre_json(instance.feedback)


class ApplyJobAdmin(admin.ModelAdmin):
    list_display = (
        "id",
        "project__owner",
        "project__name",
        "status",
        "created_at",
        "updated_at",
    )

    list_filter = ("status", "updated_at")
    ordering = ("-updated_at",)
    search_fields = (
        "project__name__iexact",
        "project__owner__username__iexact",
        "deltas_to_apply__id__startswith",
        "id",
    )
    readonly_fields = (
        "created_at",
        "updated_at",
        "started_at",
        "finished_at",
    )
    inlines = [
        DeltaInline,
    ]

    def project__owner(self, instance):
        return model_admin_url(instance.project.owner)

    project__owner.admin_order_field = "project__owner"

    def project__name(self, instance):
        return model_admin_url(instance.project, instance.project.name)

    project__name.admin_order_field = "project__name"

    def has_add_permission(self, request, obj=None):
        return False

    def has_change_permission(self, request, obj=None):
        return False

    def has_delete_permission(self, request, obj=None):
        return False


class ApplyJobDeltaInline(admin.TabularInline):
    model = ApplyJobDelta

    readonly_fields = (
        "job_id",
        "status",
        "output__pre",
    )

    fields = (
        "job_id",
        "status",
        "output__pre",
    )

    def job_id(self, instance):
        return model_admin_url(instance.apply_job)

    def output__pre(self, instance):
        return format_pre_json(instance.output)

    def has_add_permission(self, request, obj):
        return False

    def has_delete_permission(self, request, obj):
        return False


class DeltaAdmin(admin.ModelAdmin):
    list_display = (
        "id",
        "deltafile_id",
        "project__owner",
        "project__name",
        "last_status",
        "created_by",
        "created_at",
        "updated_at",
    )
    list_filter = ("last_status", "updated_at")

    actions = (
        "set_status_pending",
        "set_status_ignored",
        "set_status_unpermitted",
        "apply_selected_deltas",
    )

    readonly_fields = (
        "project",
        "deltafile_id",
        "last_feedback__pre",
        "last_modified_pk",
        "created_by",
        "created_at",
        "updated_at",
    )
    fields = (
        "project",
        "deltafile_id",
        "last_status",
        "created_by",
        "created_at",
        "updated_at",
        "content",
        "last_feedback__pre",
        "last_modified_pk",
    )
    search_fields = (
        "project__name__iexact",
        "project__owner__username__iexact",
        "last_feedback__icontains",
        "deltafile_id__startswith",
        "id",
    )
    ordering = ("-updated_at",)

    inlines = [
        ApplyJobDeltaInline,
    ]

    formfield_overrides = {JSONField: {"widget": PrettyJSONWidget}}

    change_form_template = "admin/delta_change_form.html"

    # This will disable add functionality
    def has_add_permission(self, request):
        return False

    def last_feedback__pre(self, instance):
        return format_pre_json(instance.last_feedback)

    def project__owner(self, instance):
        return model_admin_url(instance.project.owner)

    project__owner.admin_order_field = "project__owner"

    def project__name(self, instance):
        return model_admin_url(instance.project, instance.project.name)

    project__name.admin_order_field = "project__name"

    def set_status_pending(self, request, queryset):
        queryset.update(last_status=Delta.Status.PENDING)

    def set_status_ignored(self, request, queryset):
        queryset.update(last_status=Delta.Status.IGNORED)

    def set_status_unpermitted(self, request, queryset):
        queryset.update(last_status=Delta.Status.UNPERMITTED)

    def response_change(self, request, delta):
        if "_apply_delta_btn" in request.POST:
            if not delta.project.project_filename:
                self.message_user(request, "Missing project file")
                raise exceptions.NoQGISProjectError()

            if not jobs.apply_deltas(
                delta.project,
                request.user,
                delta.project.project_filename,
                delta.project.overwrite_conflicts,
                delta_ids=[str(delta.id)],
            ):
                self.message_user(request, "No deltas to apply")
                raise exceptions.NoDeltasToApplyError()

            self.message_user(request, "Delta application started")

            # we need to sleep 1 second, just to make sure the apply delta started
            time.sleep(1)

            return HttpResponseRedirect(".")

        return super().response_change(request, delta)


class PackageJobAdmin(admin.ModelAdmin):
    list_display = (
        "id",
        "project__owner",
        "project__name",
        "status",
        "created_at",
        "updated_at",
    )
    list_filter = ("status", "updated_at")
    list_select_related = ("project", "project__owner")
    actions = None
    exclude = ("feedback", "output")

    readonly_fields = (
        "project",
        "status",
        "created_at",
        "updated_at",
        "started_at",
        "finished_at",
        "output__pre",
        "feedback__pre",
    )

    search_fields = (
        "id",
        "feedback__icontains",
        "project__name__iexact",
        "project__owner__username__iexact",
    )

    ordering = ("-updated_at",)

    def project__owner(self, instance):
        return model_admin_url(instance.project.owner)

    project__owner.admin_order_field = "project__owner"

    def project__name(self, instance):
        return model_admin_url(instance.project, instance.project.name)

    project__name.admin_order_field = "project__name"

    def output__pre(self, instance):
        return format_pre(instance.output)

    def feedback__pre(self, instance):
        return format_pre_json(instance.feedback)

    # This will disable add functionality
    def has_add_permission(self, request):
        return False

    def has_change_permission(self, request, obj=None):
        return False


class ProcessProjectfileJobAdmin(admin.ModelAdmin):
    list_display = (
        "id",
        "project__owner",
        "project__name",
        "status",
        "created_at",
        "updated_at",
    )
    list_filter = ("status", "updated_at")
    list_select_related = ("project", "project__owner")
    actions = None
    exclude = ("feedback", "output")

    readonly_fields = (
        "project",
        "status",
        "created_at",
        "updated_at",
        "started_at",
        "finished_at",
        "output__pre",
        "feedback__pre",
    )

    search_fields = (
        "id",
        "feedback__icontains",
        "project__name__iexact",
        "project__owner__username__iexact",
    )

    ordering = ("-updated_at",)

    def project__owner(self, instance):
        return model_admin_url(instance.project.owner)

    project__owner.admin_order_field = "project__owner"

    def project__name(self, instance):
        return model_admin_url(instance.project, instance.project.name)

    project__name.admin_order_field = "project__name"

    def output__pre(self, instance):
        return format_pre(instance.output)

    def feedback__pre(self, instance):
        return format_pre_json(instance.feedback)

    # This will disable add functionality
    def has_add_permission(self, request):
        return False

    def has_change_permission(self, request, obj=None):
        return False


class GeodbAdmin(admin.ModelAdmin):
    list_filter = ("created_at", "hostname")
    list_display = (
        "user",
        "username",
        "dbname",
        "hostname",
        "port",
        "created_at",
        "size",
    )

    fields = (
        "user",
        "username",
        "dbname",
        "hostname",
        "port",
        "created_at",
        "size",
        "last_geodb_error",
    )

    readonly_fields = ("size", "created_at", "last_geodb_error")

    search_fields = (
        "user__username",
        "username",
        "dbname",
        "hostname",
    )

    def save_model(self, request, obj, form, change):
        # Only on creation
        if not change:
            messages.add_message(
                request,
                messages.WARNING,
                "The password is (shown only once): {}".format(obj.password),
            )
        super().save_model(request, obj, form, change)


class OrganizationMemberInline(admin.TabularInline):
    model = OrganizationMember
    fk_name = "organization"
    extra = 0


class TeamInline(admin.TabularInline):
    model = Team
    fk_name = "team_organization"
    extra = 0

    fields = ("username",)

    def has_add_permission(self, request, obj):
        return False

    def has_delete_permission(self, request, obj):
        return False

    def has_change_permission(self, request, obj):
        return False


class OrganizationAdmin(admin.ModelAdmin):
    inlines = (
        UserAccountInline,
        GeodbInline,
        OrganizationMemberInline,
        ProjectInline,
        TeamInline,
    )
    fields = (
        "username",
        "email",
        "organization_owner",
    )
    list_display = (
        "username",
        "email",
        "organization_owner__link",
        "date_joined",
        "useraccount",
    )

    search_fields = (
        "username__icontains",
        "organization_owner__username__icontains",
    )

    list_filter = ("date_joined",)

    def organization_owner__link(self, instance):
        return model_admin_url(
            instance.organization_owner, instance.organization_owner.username
        )

    def get_search_results(self, request, queryset, search_term):
        filters = search_parser(
            request,
            queryset,
            search_term,
            {
                "owner": {
                    "filter": "organization_owner__username__iexact",
                },
                "member": {
                    "filter": "membership_roles__user__username__iexact",
                },
            },
        )

        if filters:
            return queryset.filter(**filters), True

        queryset, use_distinct = super().get_search_results(
            request, queryset, search_term
        )

        return queryset, use_distinct


class TeamMemberInline(admin.TabularInline):
    model = TeamMember
    fk_name = "team"
    extra = 0


class TeamAdmin(admin.ModelAdmin):
    inlines = (TeamMemberInline,)

    list_display = (
        "username",
        "date_joined",
    )

    fields = (
        "username",
        "team_organization",
    )

    search_fields = ("username__icontains", "team_organization__username__iexact")

    list_filter = ("date_joined",)

    def save_model(self, request, obj, form, change):
        if not obj.username.startswith("@"):
            obj.username = f"@{obj.team_organization.username}/{obj.username}"
        obj.save()


admin.site.register(User, UserAdmin)
admin.site.register(Organization, OrganizationAdmin)
admin.site.register(Team, TeamAdmin)
admin.site.register(Project, ProjectAdmin)
admin.site.register(Delta, DeltaAdmin)
admin.site.register(ApplyJob, ApplyJobAdmin)
admin.site.register(PackageJob, PackageJobAdmin)
admin.site.register(ProcessProjectfileJob, ProcessProjectfileJobAdmin)
admin.site.register(Geodb, GeodbAdmin)

admin.site.unregister(Group)
admin.site.unregister(SocialAccount)
admin.site.unregister(SocialApp)
admin.site.unregister(SocialToken)


# docker-app/qfieldcloud/core/apps.py

from django.apps import AppConfig


class CoreConfig(AppConfig):
    name = "qfieldcloud.core"

    def ready(self):
        from qfieldcloud.core import signals  # noqa


# docker-app/qfieldcloud/core/converters.py

from django.urls.converters import StringConverter


class IStringConverter(StringConverter):
    def to_python(self, value):
        return value.lower()


# docker-app/qfieldcloud/core/cron.py

import logging
import os
from datetime import timedelta

from django.conf import settings
from django.utils import timezone
from django_cron import CronJobBase, Schedule
from invitations.utils import get_invitation_model
from sentry_sdk import capture_message

from ..core.models import Job, Project
from ..core.utils2 import storage
from .invitations_utils import send_invitation

logger = logging.getLogger(__name__)

QGIS_CONTAINER_NAME = os.environ.get("QGIS_CONTAINER_NAME", None)

assert QGIS_CONTAINER_NAME


class DeleteExpiredInvitationsJob(CronJobBase):
    schedule = Schedule(run_every_mins=60)
    code = "qfieldcloud.delete_expired_confirmations"

    def do(self):
        Invitation = get_invitation_model()
        Invitation.objects.delete_expired_confirmations()


class ResendFailedInvitationsJob(CronJobBase):
    schedule = Schedule(run_every_mins=60)
    code = "qfieldcloud.resend_failed_invitations"

    def do(self):
        Invitation = get_invitation_model()

        invitation_emails = []
        for invitation in Invitation.objects.filter(sent__isnull=True):
            try:
                send_invitation(invitation)
                invitation_emails.append(invitation.email)
            except Exception as err:
                logger.error(err)

        logger.info(
            f'Resend {len(invitation_emails)} previously failed invitation(s) to: {", ".join(invitation_emails)}'
        )


class SetTerminatedWorkersToFinalStatusJob(CronJobBase):
    # arbitrary number 3 here, it just feel a good number since the configuration is 10 mins
    schedule = Schedule(run_every_mins=3)
    code = "qfieldcloud.set_terminated_workers_to_final_status"

    def do(self):
        jobs = Job.objects.filter(
            status__in=[Job.Status.QUEUED, Job.Status.STARTED],
            # add extra seconds just to make sure a properly finished job properly updated the status.
            started_at__lt=timezone.now()
            - timedelta(seconds=settings.WORKER_TIMEOUT_S + 10),
        )

        for job in jobs:
            capture_message(
                f'Job "{job.id}" was with status "{job.status}", but worker container no longer exists. Job unexpectedly terminated.'
            )

        jobs.update(
            status=Job.Status.FAILED,
            finished_at=timezone.now(),
            feedback={
                "error_stack": "",
                "error": "Job unexpectedly terminated.",
                "error_origin": "worker_wrapper",
                "container_exit_code": -2,
            },
            output="Job unexpectedly terminated.",
        )


class DeleteObsoleteProjectPackagesJob(CronJobBase):
    schedule = Schedule(run_every_mins=60)
    code = "qfieldcloud.delete_obsolete_project_packages"

    def do(self):
        # get only the projects updated in the last a little more than an hour,
        # as we assume the rest of the projects have been cleaned from obsolete
        # packages in the previous CRON run. Also give 5 minute offset from now,
        # so there is no overlap and errors with the deletion happening from the
        # worker-wrapper.
        projects = Project.objects.filter(
            updated_at__gt=timezone.now() - timedelta(minutes=70),
            updated_at__lt=timezone.now() - timedelta(minutes=5),
        )
        job_ids = [
            str(job["id"])
            for job in Job.objects.filter(
                type=Job.Type.PACKAGE,
            )
            .exclude(
                status__in=(Job.Status.FAILED, Job.Status.FINISHED),
            )
            .values("id")
        ]

        for project in projects:
            project_id = str(project.id)
            package_ids = storage.get_stored_package_ids(project_id)

            for package_id in package_ids:
                # keep the last package
                if package_id == str(project.last_package_job_id):
                    continue

                # the job is still active, so it might be one of the new packages
                if package_id in job_ids:
                    continue

                storage.delete_stored_package(project_id, package_id)


# docker-app/qfieldcloud/core/exceptions.py

from rest_framework import status


class QFieldCloudException(Exception):
    """Generic QFieldCloud Exception"""

    code = "unknown_error"
    message = "QFieldcloud Unknown Error"
    status_code = None

    def __init__(self, detail="", status_code=None):

        self.detail = detail

        if status_code:
            self.status_code = status_code
        elif self.status_code:
            pass
        else:
            self.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR

        super().__init__(self.message)

    def __str__(self):
        return self.message


class StatusNotOkError(QFieldCloudException):
    """Raised when some parts of QFieldCloud are not working as expected"""

    code = "status_not_ok"
    message = "Status not ok"
    status_code = status.HTTP_503_SERVICE_UNAVAILABLE


class AuthenticationFailedError(QFieldCloudException):
    """Raised when QFieldCloud incoming request includes incorrect authentication."""

    code = "authentication_failed"
    message = "Authentication failed"
    status_code = status.HTTP_401_UNAUTHORIZED


class NotAuthenticatedError(QFieldCloudException):
    """Raised when QFieldCloud unauthenticated request fails the permission checks."""

    code = "not_authenticated"
    message = "Not authenticated"
    status_code = status.HTTP_401_UNAUTHORIZED


class EmptyContentError(QFieldCloudException):
    """Raised when a request doesn't contain an expected content
    (e.g. a file)"""

    code = "empty_content"
    message = "Empty content"
    status_code = status.HTTP_503_SERVICE_UNAVAILABLE


class ObjectNotFoundError(QFieldCloudException):
    """Raised when a requested object doesn't exist
    (e.g. wrong project id into the request)"""

    code = "object_not_found"
    message = "Object not found"
    status_code = status.HTTP_400_BAD_REQUEST


class APIError(QFieldCloudException):
    """Raised in case of an API error"""

    code = "api_error"
    message = "API Error"
    status_code = status.HTTP_500_INTERNAL_SERVER_ERROR


class ValidationError(QFieldCloudException):
    """Raised when validation of form data or model field fails
    (e.g. wrong field in request object)"""

    code = "validation_error"
    message = "Validation error"
    status_code = status.HTTP_400_BAD_REQUEST


class MultipleProjectsError(QFieldCloudException):
    """Raised when the user is trying to upload more than one QGIS project
    into a QFieldCloud project"""

    code = "multiple_projects"
    message = "Multiple projects"
    status_code = status.HTTP_400_BAD_REQUEST


class DeltafileValidationError(QFieldCloudException):
    """Raised when a deltafile validation fails"""

    code = "invalid_deltafile"
    message = "Invalid deltafile"
    status_code = status.HTTP_400_BAD_REQUEST


class NoDeltasToApplyError(QFieldCloudException):
    """Raised when a deltafile validation fails"""

    code = "no_deltas_to_apply"
    message = "No deltas to apply"
    status_code = status.HTTP_400_BAD_REQUEST


class NoQGISProjectError(QFieldCloudException):
    """Raised when a QFieldCloud doesn't contain a QGIS project that is needed
    for the requested operation"""

    code = "no_qgis_project"
    message = "The project does not contain a valid QGIS project file"
    status_code = status.HTTP_400_BAD_REQUEST


class InvalidJobError(QFieldCloudException):
    """Raised when a requested job doesn't exist"""

    code = "invalid_job"
    message = "Invalid job"
    status_code = status.HTTP_400_BAD_REQUEST


class QGISPackageError(QFieldCloudException):
    """Raised when the QGIS package of a project fails"""

    code = "qgis_package_error"
    message = "QGIS package failed"
    status_code = status.HTTP_500_INTERNAL_SERVER_ERROR

    if "Unable to open file with QGIS" in message:
        message = "QGIS is unable to open the QGIS project"


class QuotaError(QFieldCloudException):
    """Raised when a quota limitation is hit"""

    code = "over_quota"
    message = "Quota error"
    status_code = status.HTTP_402_PAYMENT_REQUIRED


# docker-app/qfieldcloud/core/geodb_utils.py

import os

import psycopg2
from psycopg2 import sql
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT


class GeodbConnection(object):
    def __init__(self):
        pass

    def __enter__(self):
        host = os.environ.get("GEODB_HOST")
        port = os.environ.get("GEODB_PORT")

        # If geodb is running on the same machine we connect trough
        # the internal docker net
        if host == "geodb":
            port = 5432

        self.connection = psycopg2.connect(
            dbname=os.environ.get("GEODB_DB"),
            user=os.environ.get("GEODB_USER"),
            password=os.environ.get("GEODB_PASSWORD"),
            host=host,
            port=port,
        )

        return self.connection

    def __exit__(self, type, value, traceback):
        self.connection.close()


def geodb_is_running():
    """Check the connection to the geodb"""

    try:
        with GeodbConnection():
            pass
    except psycopg2.Error:
        return False

    return True


def create_role_and_db(geodb):
    """Create role and db.
    This function is automatically called when a Geodb object is created
    """

    with GeodbConnection() as conn:

        # CREATE DATABASE cannot run inside a transaction block
        conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)

        cur = conn.cursor()

        cur.execute(
            sql.SQL(
                """
            CREATE ROLE {} WITH
            LOGIN
            NOSUPERUSER
            NOCREATEDB
            NOCREATEROLE
            INHERIT
            NOREPLICATION
            CONNECTION LIMIT 5
            PASSWORD %s;
            """
            ).format(sql.Identifier(geodb.username)),
            (geodb.password,),
        )

        cur.execute(
            sql.SQL(
                """
            CREATE DATABASE {}
            WITH
            OWNER = %s
            TEMPLATE = template_postgis
            ENCODING = 'UTF8'
            CONNECTION LIMIT = 5;
            """
            ).format(sql.Identifier(geodb.dbname)),
            (geodb.username,),
        )

    result = {
        "hostname": geodb.hostname,
        "port": geodb.port,
        "username": geodb.username,
        "dbname": geodb.dbname,
        "password": geodb.password,
    }
    return result


def delete_db_and_role(dbname, username):
    with GeodbConnection() as conn:
        # DROP DATABASE cannot be executed inside a transaction block
        conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)

        cur = conn.cursor()

        cur.execute(
            sql.SQL(
                """
            DROP DATABASE IF EXISTS {};
            """
            ).format(sql.Identifier(dbname))
        )

        cur.execute(
            sql.SQL(
                """
            DROP ROLE IF EXISTS {};
            """
            ).format(sql.Identifier(username))
        )


def get_db_size(geodb):
    """Return the size of the database in bytes"""

    with GeodbConnection() as conn:
        cur = conn.cursor()
        cur.execute("SELECT pg_database_size(%s);", (geodb.dbname,))
        return cur.fetchone()[0]


# docker-app/qfieldcloud/core/invitations_utils.py

import logging

from django.contrib.sites.models import Site
from django.core.exceptions import ValidationError
from django.core.validators import validate_email
from django.urls import reverse
from django.utils import timezone
from django.utils.translation import gettext as _
from invitations.adapters import get_invitations_adapter
from invitations.signals import invite_url_sent
from invitations.utils import get_invitation_model
from qfieldcloud.core import permissions_utils
from qfieldcloud.core.models import User

logger = logging.getLogger(__name__)


def is_valid_email(email: str) -> bool:
    try:
        validate_email(email)
        return True
    except ValidationError:
        return False


def invite_user_by_email(email: str, inviter: User, send: bool = True) -> tuple:
    """
    Sends an invite for a given email address

    Args :
        email:      recipient's email
        inviter:    inviter
        send:       whether the email should be sent

    Returns :
        (bool, str) success, message
    """

    try:
        validate_email(email)
    except ValidationError:
        return False, _(
            "The provided email address is not valid. Users can only be invited with a valid email address."
        )

    if not permissions_utils.can_send_invitations(inviter, inviter):
        return False, _("Your user cannot send invitations.")

    Invitation = get_invitation_model()
    Invitation.objects.filter(email=email, sent__isnull=True).delete()
    qs = Invitation.objects.filter(email=email)

    if len(qs) > 0:
        invite = qs[0]

        assert len(qs) == 1

        if invite.key_expired():
            qs.delete()
        else:
            return (
                False,
                _(
                    "{} has already been invited to create a QFieldCloud account."
                ).format(email),
            )

    qs = User.objects.filter(email=email, user_type=User.TYPE_USER)

    if len(qs) > 0:
        assert len(qs) == 1

        return (
            False,
            _("{} has already been used by a registered QFieldCloud user.").format(
                email
            ),
        )

    invite = Invitation.create(email, inviter=inviter)

    # TODO see if we can "pre-attach" this future user to this project, probably to be done
    # at the same time as TODO below about actual invitation
    if send and inviter.remaining_invitations > 0:
        inviter.remaining_invitations -= 1
        inviter.save()
        send_invitation(invite)
        return (
            True,
            _("{} has been invited to create a QFieldCloud account.").format(email),
        )

    return True, _("{} has been added to the QFieldCloud waiting list.").format(email)


def send_invitation(invite, **kwargs):
    """Sends invitation.

    Args:
        invite (Invitation): the invitation to be sent

    The same as the original Invitation.send_invitation, but without passing the request object.
    https://github.com/bee-keeper/django-invitations/blob/invitations/models.py#L42
    """
    current_site = kwargs.pop("site", Site.objects.get_current())
    invite_url = reverse("invitations:accept-invite", args=[invite.key])
    invite_url = f"https://{current_site.domain}{invite_url}"
    ctx = kwargs
    ctx.update(
        {
            "invite_url": invite_url,
            "site_name": current_site.name,
            "email": invite.email,
            "key": invite.key,
            "inviter": invite.inviter,
        }
    )

    email_template = "invitations/email/email_invite"

    get_invitations_adapter().send_mail(
        email_template,
        invite.email,
        ctx,
    )
    invite.sent = timezone.now()
    invite.save()

    invite_url_sent.send(
        sender=invite.__class__,
        instance=invite,
        invite_url_sent=invite_url,
        inviter=invite.inviter,
    )


# docker-app/qfieldcloud/core/models.py

import calendar
import contextlib
import os
import secrets
import string
import uuid
from datetime import date, timedelta
from enum import Enum
from typing import List, Optional

import django_cryptography.fields
import qfieldcloud.core.utils2.storage
from auditlog.registry import auditlog
from deprecated import deprecated
from django.contrib.auth.models import AbstractUser, UserManager
from django.contrib.gis.db import models
from django.core.exceptions import ValidationError
from django.core.validators import MinValueValidator, RegexValidator
from django.db import transaction
from django.db.models import Case, Exists, F, OuterRef, Q
from django.db.models import Value as V
from django.db.models import When
from django.db.models.aggregates import Count, Sum
from django.db.models.fields.json import JSONField
from django.urls import reverse_lazy
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import gettext as _
from model_utils.managers import InheritanceManager
from qfieldcloud.core import geodb_utils, utils, validators
from qfieldcloud.subscription.models import Plan
from timezone_field import TimeZoneField

# http://springmeblog.com/2018/how-to-implement-multiple-user-types-with-django/


class UserQueryset(models.QuerySet):
    """Adds for_project(user) method to the user's querysets, allowing to filter only users part of a project.

    Users are annotated with the user's project role (`project_role`) and the origin of this role (`project_role_origin`).
    If the project is public, it will return only the directly collaborated users.

    Args:
        project:               project to find users for

    Usage:
    ```
    # List all users that are involved in OpenKebabMap.
    Users.object.for_project(OpenKebabMap)
    ```

    Note:
    This query is very similar to `ProjectQueryset.for_user`, don't forget to update it too.
    """

    def for_project(self, project: "Project"):
        qs = (
            self.defer("project_roles__project_id", "project_roles__project_id")
            .filter(
                user_type=User.TYPE_USER,
                project_roles__project=project,
                project_roles__is_valid=True,
            )
            .annotate(
                project_role=F("project_roles__name"),
                project_role_origin=F("project_roles__origin"),
            )
        )

        return qs

    def for_organization(self, organization: "Organization"):
        qs = (
            self.defer(
                "organization_roles__user_id",
                "organization_roles__organization_id",
            )
            .filter(
                user_type=User.TYPE_USER,
                organization_roles__organization=organization,
            )
            .annotate(
                organization_role=F("organization_roles__name"),
                organization_role_origin=F("organization_roles__origin"),
                organization_role_is_public=F("organization_roles__is_public"),
            )
        )

        return qs

    def for_team(self, team: "Team"):
        permissions_config = [
            # Direct ownership of the organization
            (
                Exists(
                    Team.objects.filter(
                        pk=team.pk,
                        team_organization__organization_owner=OuterRef("pk"),
                    )
                ),
                V(TeamMember.Roles.ADMIN),
            ),
            # Team membership
            (
                Exists(
                    TeamMember.objects.filter(
                        team=team,
                        member=OuterRef("pk"),
                    )
                ),
                V(TeamMember.Roles.MEMBER),
            ),
        ]

        qs = self.annotate(
            membership_role=Case(
                *[When(perm[0], perm[1]) for perm in permissions_config],
                default=None,
                output_field=models.CharField(),
            ),
        )
        qs = qs.exclude(membership_role__isnull=True)

        return qs

    def for_entity(self, entity: "User"):
        """Returns all users grouped in given entity (any type)

        Internally calls for_team or for_organization depending on the entity."""

        if entity.user_type == User.TYPE_USER:
            return self.filter(pk=entity.pk)

        if entity.user_type == User.TYPE_TEAM:
            return self.for_team(entity)

        if entity.user_type == User.TYPE_ORGANIZATION:
            return self.for_organization(entity)

        raise RuntimeError(f"Unsupported entity : {entity}")


class QFieldCloudUserManager(UserManager):
    def get_queryset(self):
        return UserQueryset(self.model, using=self._db)

    def for_project(self, project):
        return self.get_queryset().for_project(project)

    def for_organization(self, organization):
        return self.get_queryset().for_organization(organization)

    def for_team(self, entity):
        return self.get_queryset().for_team(entity)

    def for_entity(self, entity):
        return self.get_queryset().for_entity(entity)


# TODO change types to Enum
class User(AbstractUser):
    """User model. Used as base for organizations and teams too.

    Args:
        AbstractUser (AbstractUser): the django's abstract user base

    Returns:
        User: the user instance

    Note:
        If you add validators in the constructor, note they will be added multiple times for each class that extends User.
    """

    objects = QFieldCloudUserManager()

    TYPE_USER = 1
    TYPE_ORGANIZATION = 2
    TYPE_TEAM = 3

    TYPE_CHOICES = (
        (TYPE_USER, "user"),
        (TYPE_ORGANIZATION, "organization"),
        (TYPE_TEAM, "team"),
    )

    """Define username here, so we can avoid multiple validators from the constructor. Check the class notes."""
    username = models.CharField(
        _("username"),
        max_length=150,
        unique=True,
        help_text=_(
            "Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only."
        ),
        validators=[
            RegexValidator(
                r"^[-a-zA-Z0-9_]+$",
                "Only letters, numbers, underscores or hyphens are allowed.",
            ),
            RegexValidator(r"^[a-zA-Z].*$", _("The name must begin with a letter.")),
            RegexValidator(
                r"^.{3,}$", _("The name must be at least 3 characters long.")
            ),
            validators.reserved_words_validator,
        ],
        error_messages={
            "unique": _("A user with that username already exists."),
        },
    )

    user_type = models.PositiveSmallIntegerField(
        choices=TYPE_CHOICES, default=TYPE_USER, editable=False
    )

    remaining_invitations = models.PositiveIntegerField(
        default=3,
        help_text=_("Remaining invitations that can be sent by the user himself."),
    )

    has_newsletter_subscription = models.BooleanField(default=False)
    has_accepted_tos = models.BooleanField(default=False)

    def __str__(self):
        return self.username

    def get_absolute_url(self):
        if self.user_type == User.TYPE_TEAM:
            team = Team.objects.get(pk=self.pk)
            return reverse_lazy(
                "settings_teams_edit",
                kwargs={
                    "username": team.team_organization.username,
                    "teamname": team.teamname,
                },
            )
        else:
            return reverse_lazy("profile_overview", kwargs={"username": self.username})

    @property
    def is_user(self):
        return self.user_type == User.TYPE_USER

    @property
    def is_organization(self):
        return self.user_type == User.TYPE_ORGANIZATION

    @property
    def is_team(self):
        return self.user_type == User.TYPE_TEAM

    @property
    def full_name(self) -> str:
        return f"{self.first_name} {self.last_name}".strip()

    @property
    def username_with_full_name(self) -> str:
        full_name = self.full_name.strip()

        if full_name:
            return f"{self.username} ({full_name})"
        else:
            return self.username

    @property
    def has_geodb(self) -> bool:
        return hasattr(self, "geodb")

    def save(self, *args, **kwargs):
        # if the user is created, we need to create a user account
        if self._state.adding and self.user_type != User.TYPE_TEAM:
            with transaction.atomic():
                super().save(*args, **kwargs)
                plan = Plan.objects.get(user_type=self.user_type, is_default=True)
                UserAccount.objects.create(user=self, plan=plan)
        else:
            super().save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        if self.user_type != User.TYPE_TEAM:
            qfieldcloud.core.utils2.storage.remove_user_avatar(self)

        with no_audits([User, UserAccount, Project]):
            super().delete(*args, **kwargs)

    class Meta:
        verbose_name = "user"
        verbose_name_plural = "users"

    #     # TODO Django 4.0 convert to functional unique constraint, see https://docs.djangoproject.com/en/4.0/ref/models/constraints/#expressions
    #     constraints = [
    #         # the uniqueness is guaranteed by username's unqiue index anyway
    #         # since Django has some annoying checks if `username` is unique, better sacrifice
    #         # some megabytes with two separate indexes for `username` and `UPPER(username)`
    #         UniqueConstraint(
    #             Upper("username"),
    #             name="core_user_username_uppercase"
    #         )
    #     ]


class UserAccount(models.Model):

    NOTIFS_IMMEDIATELY = timedelta(minutes=0)
    NOTIFS_HOURLY = timedelta(hours=1)
    NOTIFS_DAILY = timedelta(days=1)
    NOTIFS_WEEKLY = timedelta(weeks=1)
    NOTIFS_DISABLED = None
    NOTIFS_CHOICES = (
        (NOTIFS_IMMEDIATELY, _("Immediately")),
        (NOTIFS_HOURLY, _("Hourly")),
        (NOTIFS_DAILY, _("Daily")),
        (NOTIFS_WEEKLY, _("Weekly")),
        (NOTIFS_DISABLED, _("Disabled")),
    )

    user = models.OneToOneField(User, on_delete=models.CASCADE, primary_key=True)

    plan = models.ForeignKey(
        "subscription.Plan",
        on_delete=models.PROTECT,
    )

    # These will be moved one day to extrapackage. We don't touch for now (they are only used
    # in some tests)
    db_limit_mb = models.PositiveIntegerField(default=25)
    is_geodb_enabled = models.BooleanField(
        default=False,
        help_text=_("Whether the account has the option to create a GeoDB."),
    )

    bio = models.CharField(max_length=255, default="", blank=True)
    company = models.CharField(max_length=255, default="", blank=True)
    location = models.CharField(max_length=255, default="", blank=True)
    twitter = models.CharField(max_length=255, default="", blank=True)
    is_email_public = models.BooleanField(default=False)
    avatar_uri = models.CharField(_("Profile Picture URI"), max_length=255, blank=True)
    timezone = TimeZoneField(default="Europe/Zurich", choices_display="WITH_GMT_OFFSET")

    notifs_frequency = models.DurationField(
        verbose_name=_("Email frequency for notifications"),
        choices=NOTIFS_CHOICES,
        default=NOTIFS_DISABLED,
        null=True,
        blank=True,
    )

    @property
    def avatar_url(self):
        if self.avatar_uri:
            return reverse_lazy(
                "public_files",
                kwargs={"filename": self.avatar_uri},
            )
        else:
            return None

    @property
    def storage_quota_total_mb(self) -> float:
        """Returns the storage quota left in MB (quota from account and extrapackages minus storage of all owned projects)"""

        base_quota = self.plan.storage_mb

        extra_quota = (
            self.extra_packages.filter(
                Q(start_date__lte=timezone.now())
                & (Q(end_date__isnull=True) | Q(end_date__gte=timezone.now()))
            ).aggregate(sum_mb=Sum("type__extrapackagetypestorage__megabytes"))[
                "sum_mb"
            ]
            or 0
        )

        return base_quota + extra_quota

    @property
    def storage_quota_used_mb(self) -> float:
        """Returns the storage used in MB"""
        used_quota = (
            self.user.projects.aggregate(sum_mb=Sum("storage_size_mb"))["sum_mb"] or 0
        )

        return used_quota

    @property
    def storage_quota_left_mb(self) -> float:
        """Returns the storage quota left in MB (quota from account and extrapackages minus storage of all owned projects)"""

        return self.storage_quota_total_mb - self.storage_quota_used_mb

    @property
    def storage_quota_used_perc(self) -> float:
        """Returns the storage used in percentage (%) of the total storage"""
        return max(
            0, min(self.storage_quota_used_mb / self.storage_quota_total_mb * 100, 100)
        )

    def __str__(self):
        return f"Account {self.plan}"


class Geodb(models.Model):
    def random_string():
        """Generate random sting starting with a lowercase letter and then
        lowercase letters and digits"""

        first_letter = secrets.choice(string.ascii_lowercase)
        letters_and_digits = string.ascii_lowercase + string.digits
        secure_str = first_letter + "".join(
            (secrets.choice(letters_and_digits) for i in range(15))
        )
        return secure_str

    def random_password():
        """Generate secure random password composed of
        letters, digits and special characters"""

        password_characters = (
            string.ascii_letters + string.digits + "!#$%&()*+,-.:;<=>?@[]_{}~"
        )
        secure_str = "".join((secrets.choice(password_characters) for i in range(16)))
        return secure_str

    def default_hostname():
        return os.environ.get("GEODB_HOST")

    def default_port():
        return os.environ.get("GEODB_PORT")

    user = models.OneToOneField(User, on_delete=models.CASCADE, primary_key=True)

    username = models.CharField(blank=False, max_length=255, default=random_string)
    dbname = models.CharField(blank=False, max_length=255, default=random_string)
    hostname = models.CharField(blank=False, max_length=255, default=default_hostname)
    port = models.PositiveIntegerField(default=default_port)
    created_at = models.DateTimeField(auto_now_add=True)

    # The password is generated but not stored into the db
    password = ""
    last_geodb_error = None

    def __init__(self, *args, password="", **kwargs) -> None:
        super().__init__(*args, **kwargs)

        self.password = password

        if not self.password:
            self.password = Geodb.random_password()

    def size(self):
        try:
            return geodb_utils.get_db_size(self)
        except Exception as err:
            self.last_geodb_error = str(err)
            return None

    def __str__(self):
        return "{}'s db account, dbname: {}, username: {}".format(
            self.user.username, self.dbname, self.username
        )

    def save(self, *args, **kwargs):
        created = self._state.adding
        super().save(*args, **kwargs)
        # Automatically create a role and database when a Geodb object is created.
        if created:
            geodb_utils.create_role_and_db(self)

    def delete(self, *args, **kwargs):
        super().delete(*args, **kwargs)
        # Automatically delete role and database when a Geodb object is deleted.
        geodb_utils.delete_db_and_role(self.dbname, self.username)


class OrganizationQueryset(models.QuerySet):
    """Adds of_user(user) method to the organization's querysets, allowing to filter only organization related to that user.

    Organizations are annotated with the user's role (`membership_role`), the origin of this role (`membership_role_origin`)
    and whether it is public (`membership_is_public`).

    Args:
        user:               user to check membership for
    """

    class RoleOrigins(models.TextChoices):
        ORGANIZATIONOWNER = "organization_owner", _("Organization owner")
        ORGANIZATIONMEMBER = "organization_member", _("Organization member")

    def of_user(self, user):
        qs = (
            self.defer("membership_roles__user_id", "membership_roles__organization_id")
            .filter(
                membership_roles__user=user,
            )
            .annotate(
                membership_role=F("membership_roles__name"),
                membership_role_origin=F("membership_roles__origin"),
                membership_role_is_public=F("membership_roles__is_public"),
            )
        )

        return qs


class OrganizationManager(UserManager):
    def get_queryset(self):
        return OrganizationQueryset(self.model, using=self._db)

    def of_user(self, user):
        return self.get_queryset().of_user(user)


class Organization(User):
    objects = OrganizationManager()

    organization_owner = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        related_name="owner",
        limit_choices_to=models.Q(user_type=User.TYPE_USER),
    )

    class Meta:
        verbose_name = "organization"
        verbose_name_plural = "organizations"

    def billable_users(self, from_date: date, to_date: Optional[date] = None):
        """Returns the queryset of billable users in the given time interval.

        Billable users are users triggering a job or pushing a delta on a project owned by the organization.

        Args:
            from_date (datetime.date): inclusive beginning of the interval
            to_date (Optional[datetime.date], optional): inclusive end of the interval (if None, will default to the last day of the month of the start date)
        """

        if to_date is None:
            to_date = from_date.replace(
                day=calendar.monthrange(from_date.year, from_date.month)[1]
            )

        users_with_delta = (
            Delta.objects.filter(
                project__in=self.projects.all(),
                updated_at__gte=from_date,
                updated_at__lte=to_date,
            )
            .values_list("created_by_id", flat=True)
            .distinct()
        )
        users_with_jobs = (
            Job.objects.filter(
                project__in=self.projects.all(),
                updated_at__gte=from_date,
                updated_at__lte=to_date,
            )
            .values_list("created_by_id", flat=True)
            .distinct()
        )

        return User.objects.filter(organizationmember__organization=self).filter(
            Q(id__in=users_with_delta) | Q(id__in=users_with_jobs)
        )

    def save(self, *args, **kwargs):
        self.user_type = self.TYPE_ORGANIZATION
        return super().save(*args, **kwargs)


class OrganizationMemberQueryset(models.QuerySet):
    @transaction.atomic
    def delete(self, *args, **kwargs):
        team_total, team_summary = TeamMember.objects.filter(
            team__team_organization__in=self.values_list("organization"),
            member__in=self.values_list("member"),
        ).delete()

        org_total, org_summary = super().delete(*args, **kwargs)

        total = org_total + team_total
        summary = {**org_summary, **team_summary}

        return total, summary


class OrganizationMember(models.Model):

    objects = OrganizationMemberQueryset.as_manager()

    class Roles(models.TextChoices):
        ADMIN = "admin", _("Admin")
        MEMBER = "member", _("Member")

    class Meta:
        constraints = [
            models.UniqueConstraint(
                fields=["organization", "member"],
                name="organization_organization_member_uniq",
            )
        ]

    organization = models.ForeignKey(
        Organization,
        on_delete=models.CASCADE,
        limit_choices_to=models.Q(user_type=User.TYPE_ORGANIZATION),
        related_name="members",
    )
    member = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        limit_choices_to=models.Q(user_type=User.TYPE_USER),
    )
    role = models.CharField(max_length=10, choices=Roles.choices, default=Roles.MEMBER)

    is_public = models.BooleanField(default=False)

    def __str__(self):
        return self.organization.username + ": " + self.member.username

    def clean(self) -> None:
        if self.organization.organization_owner == self.member:
            raise ValidationError(_("Cannot add the organization owner as a member."))

        max_organization_members = (
            self.organization.useraccount.plan.max_organization_members
        )
        if (
            max_organization_members > -1
            and self.organization.members.count() >= max_organization_members
        ):
            raise ValidationError(
                _(
                    "Cannot add new organization members, account limit has been reached."
                )
            )

        return super().clean()

    @transaction.atomic
    def delete(self, *args, **kwargs) -> None:
        super().delete(*args, **kwargs)

        TeamMember.objects.filter(
            team__team_organization=self.organization,
            member=self.member,
        ).delete()

    def save(self, *args, **kwargs):
        self.full_clean()
        return super().save(*args, **kwargs)


class OrganizationRolesView(models.Model):
    user = models.ForeignKey(
        User,
        on_delete=models.DO_NOTHING,
        related_name="organization_roles",
    )
    organization = models.ForeignKey(
        "Organization",
        on_delete=models.DO_NOTHING,
        related_name="membership_roles",
    )
    name = models.CharField(max_length=100, choices=OrganizationMember.Roles.choices)
    origin = models.CharField(
        max_length=100, choices=OrganizationQueryset.RoleOrigins.choices
    )
    is_public = models.BooleanField()

    class Meta:
        db_table = "organizations_with_roles_vw"
        managed = False


class Team(User):

    team_organization = models.ForeignKey(
        Organization,
        on_delete=models.CASCADE,
        related_name="teams",
    )

    class Meta:
        verbose_name = "team"
        verbose_name_plural = "teams"

    def save(self, *args, **kwargs):
        self.user_type = self.TYPE_TEAM
        return super().save(*args, **kwargs)

    @property
    def teamname(self):
        return self.username.replace(f"@{self.team_organization.username}/", "")


class TeamMember(models.Model):
    class Roles(models.TextChoices):
        ADMIN = "admin", _("Admin")
        MEMBER = "member", _("Member")

    class Meta:
        constraints = [
            models.UniqueConstraint(
                fields=["team", "member"],
                name="team_team_member_uniq",
            )
        ]

    team = models.ForeignKey(
        Team,
        on_delete=models.CASCADE,
        limit_choices_to=models.Q(user_type=User.TYPE_TEAM),
        related_name="members",
    )
    member = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        limit_choices_to=models.Q(user_type=User.TYPE_USER),
    )

    def clean(self) -> None:
        if (
            self.team.team_organization.members.filter(member=self.member).count() == 0
            and self.team.team_organization.organization_owner != self.member
        ):
            raise ValidationError(
                _("Cannot add team member that is not an organization member.")
            )

        return super().clean()

    def save(self, *args, **kwargs):
        self.full_clean()
        return super().save(*args, **kwargs)

    def __str__(self):
        return self.team.username + ": " + self.member.username


class ProjectQueryset(models.QuerySet):
    """Adds for_user(user) method to the project's querysets, allowing to filter only projects visible to that user.

    Projects are annotated with the user's role (`user_role`) and the origin of this role (`user_role_origin`).

    Args:
        user:               user to check permission for

    Usage:
    ```
    # List Olivier's projects that are visible to Ivan (olivier/ivan are User instances)
    olivier.projects.for_user(ivan)
    ```

    Note:
    This query is very similar to `UserQueryset.for_project`, don't forget to update it too.
    """

    class RoleOrigins(models.TextChoices):
        PROJECTOWNER = "project_owner", _("Project owner")
        ORGANIZATIONOWNER = "organization_owner", _("Organization owner")
        ORGANIZATIONADMIN = "organization_admin", _("Organization admin")
        COLLABORATOR = "collaborator", _("Collaborator")
        TEAMMEMBER = "team_member", _("Team member")
        PUBLIC = "public", _("Public")

    def for_user(self, user):
        qs = (
            self.defer("user_roles__user_id", "user_roles__project_id")
            .filter(
                user_roles__user=user,
                user_roles__is_valid=True,
            )
            .annotate(
                user_role=F("user_roles__name"),
                user_role_origin=F("user_roles__origin"),
            )
        )

        return qs


class Project(models.Model):
    """Represent a QFieldcloud project.
    It corresponds to a directory on the file system.

    The owner of a project is an Organization.
    """

    # NOTE the status is NOT stored in the db, because it might be refactored
    class Status(models.TextChoices):
        OK = "ok", _("Ok")
        BUSY = "busy", _("Busy")
        FAILED = "failed", _("Failed")

    class StatusCode(models.TextChoices):
        OK = "ok", _("Ok")
        FAILED_PROCESS_PROJECTFILE = "failed_process_projectfile", _(
            "Failed process projectfile"
        )
        TOO_MANY_COLLABORATORS = "too_many_collaborators", _("Too many collaborators")

    objects = ProjectQueryset.as_manager()

    _status_code = StatusCode.OK

    class Meta:
        ordering = ["owner__username", "name"]
        constraints = [
            models.UniqueConstraint(
                fields=["owner", "name"], name="project_owner_name_uniq"
            )
        ]

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(
        max_length=255,
        validators=[
            RegexValidator(
                r"^[a-zA-Z0-9-_\.]+$",
                _("Only letters, numbers, underscores, hyphens and dots are allowed."),
            )
        ],
        help_text=_(
            _("Only letters, numbers, underscores, hyphens and dots are allowed.")
        ),
    )

    description = models.TextField(blank=True)
    project_filename = models.TextField(blank=True, null=True)
    project_details = models.JSONField(blank=True, null=True)
    is_public = models.BooleanField(
        default=False,
        help_text=_(
            "Projects that are marked as public would be visible and editable to anyone."
        ),
    )
    owner = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        related_name="projects",
        limit_choices_to=models.Q(
            user_type__in=[User.TYPE_USER, User.TYPE_ORGANIZATION]
        ),
        help_text=_(
            "The project owner can be either you or any of the organization you are member of."
        ),
    )
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    # These cache stats of the S3 storage. These can be out of sync, and should be
    # refreshed whenever retrieving/uploading files by passing `project.save(recompute_storage=True)`
    storage_size_mb = models.FloatField(default=0)

    # NOTE we can track only the file based layers, WFS, WMS, PostGIS etc are impossible to track
    data_last_updated_at = models.DateTimeField(blank=True, null=True)
    data_last_packaged_at = models.DateTimeField(blank=True, null=True)

    last_package_job = models.ForeignKey(
        "PackageJob",
        on_delete=models.SET_NULL,
        related_name="last_job_of",
        null=True,
        blank=True,
    )

    repackaging_cache_expire = models.DurationField(
        default=timedelta(minutes=60),
        validators=[MinValueValidator(timedelta(minutes=1))],
    )

    overwrite_conflicts = models.BooleanField(
        default=True,
        help_text=_(
            "If enabled, QFieldCloud will automatically overwrite conflicts in this project. Disabling this will force the project manager to manually resolve all the conflicts."
        ),
    )
    thumbnail_uri = models.CharField(
        _("Thumbnail Picture URI"), max_length=255, blank=True
    )

    @property
    def thumbnail_url(self):
        if self.thumbnail_uri:
            return reverse_lazy(
                "project_metafiles",
                kwargs={"projectid": self.id, "filename": self.thumbnail_uri[51:]},
            )
        else:
            return None

    def get_absolute_url(self):
        return reverse_lazy(
            "project_overview",
            kwargs={"username": self.owner.username, "project": self.name},
        )

    def __str__(self):
        return self.name + " (" + str(self.id) + ")" + " owner: " + self.owner.username

    @property
    def attachment_dirs(self) -> List[str]:
        """Returns a list of configured attachment dirs for the project.

        Attachment dir is a special directory in the QField infrastructure that holds attachment files
        such as images, pdf etc. By default "DCIM" is considered a attachment directory.

        TODO this function expects whether `attachment_dirs` key in project_details. However,
        neither the extraction from the projectfile, nor the configuration in QFieldSync are implemented.

        Returns:
            List[str]: A list configured attachment dirs for the project.
        """
        attachment_dirs = []

        if self.project_details and self.project_details.get("attachment_dirs"):
            attachment_dirs = self.project_details.get("attachment_dirs", [])

        if not attachment_dirs:
            attachment_dirs = ["DCIM"]

        return attachment_dirs

    @property
    def private(self) -> bool:
        # still used in the project serializer
        return not self.is_public

    @cached_property
    def files(self) -> List[utils.S3ObjectWithVersions]:
        """Gets all the files from S3 storage. This is potentially slow. Results are cached on the instance."""
        return list(utils.get_project_files_with_versions(self.id))

    @property
    @deprecated("Use `len(project.files)` instead")
    def files_count(self):
        return len(self.files)

    @property
    def users(self):
        return User.objects.for_project(self)

    @property
    def has_online_vector_data(self) -> Optional[bool]:
        """Returns None if project details or layers details are not available"""

        # it's safer to assume there is an online vector layer
        if not self.project_details:
            return None

        layers_by_id = self.project_details.get("layers_by_id")

        # it's safer to assume there is an online vector layer
        if layers_by_id is None:
            return None

        has_online_vector_layers = False

        for layer_data in layers_by_id.values():
            if layer_data.get("type_name") == "VectorLayer" and not layer_data.get(
                "filename", ""
            ):
                has_online_vector_layers = True
                break

        return has_online_vector_layers

    @property
    def can_repackage(self) -> bool:
        return True

    @property
    def needs_repackaging(self) -> bool:
        if (
            # if has_online_vector_data is None (happens when the project details are missing)
            # we assume there might be
            self.has_online_vector_data is False
            and self.data_last_updated_at
            and self.data_last_packaged_at
        ):
            # if all vector layers are file based and have been packaged after the last update, it is safe to say there are no modifications
            return self.data_last_packaged_at < self.data_last_updated_at
        else:
            # if the project has online vector layers (PostGIS/WFS/etc) we cannot be sure if there are modification or not, so better say there are
            return True

    @property
    def status(self) -> Status:
        # NOTE the status is NOT stored in the db, because it might be outdated
        if (
            Job.objects.filter(
                project=self, status__in=[Job.Status.QUEUED, Job.Status.STARTED]
            ).count()
            > 0
        ):
            return Project.Status.BUSY
        else:
            status = Project.Status.OK
            status_code = Project.StatusCode.OK
            max_premium_collaborators_per_private_project = (
                self.owner.useraccount.plan.max_premium_collaborators_per_private_project
            )

            if not self.project_filename:
                status = Project.Status.FAILED
                status_code = Project.StatusCode.FAILED_PROCESS_PROJECTFILE
            elif self.is_public or (
                max_premium_collaborators_per_private_project != -1
                and max_premium_collaborators_per_private_project
                < self.direct_collaborators.count()
            ):
                status = Project.Status.FAILED
                status_code = Project.StatusCode.TOO_MANY_COLLABORATORS

            self._status_code = status_code
            return status

    @property
    def status_code(self) -> StatusCode:
        return self._status_code

    @property
    def storage_size_perc(self) -> float:
        return (
            self.storage_size_mb / self.owner.useraccount.storage_quota_total_mb * 100
        )

    @property
    def direct_collaborators(self):
        if self.owner.is_organization:
            exclude_pks = [self.owner.organization.organization_owner_id]
        else:
            exclude_pks = [self.owner_id]

        return self.collaborators.filter(
            collaborator__user_type=User.TYPE_USER,
        ).exclude(
            collaborator_id__in=exclude_pks,
        )

    def delete(self, *args, **kwargs):
        if self.thumbnail_uri:
            qfieldcloud.core.utils2.storage.remove_project_thumbail(self)
        super().delete(*args, **kwargs)

    def save(self, recompute_storage=False, *args, **kwargs):
        if recompute_storage:
            self.storage_size_mb = utils.get_s3_project_size(self.id)
        super().save(*args, **kwargs)


class ProjectCollaboratorQueryset(models.QuerySet):
    def validated(self, keep_invalid=False):
        """Annotates the queryset with `is_valid` and by default filters out all invalid memberships.

        A membership to a private project not owned by an organization, or owned by a organization
        that the member is not part of is invalid.

        Args:
            keep_invalid:   if true, invalid rows are kept"""

        # Build the conditions with Q objects
        public = Q(project__is_public=True)
        owned_by_org = Q(project__owner__user_type=User.TYPE_ORGANIZATION)
        user_also_member_of_org = Q(
            project__owner__organization__members__member=F("collaborator")
        )

        # Assemble the condition
        condition = public | (owned_by_org & user_also_member_of_org)

        # Annotate the queryset
        qs = self.annotate(is_valid=Case(When(condition, then=True), default=False))

        # Filter out invalid
        if not keep_invalid:
            qs = qs.exclude(is_valid=False)

        return qs


class ProjectCollaborator(models.Model):
    class Roles(models.TextChoices):
        ADMIN = "admin", _("Admin")
        MANAGER = "manager", _("Manager")
        EDITOR = "editor", _("Editor")
        REPORTER = "reporter", _("Reporter")
        READER = "reader", _("Reader")

    class Meta:
        constraints = [
            models.UniqueConstraint(
                fields=["project", "collaborator"],
                name="projectcollaborator_project_collaborator_uniq",
            )
        ]

    objects = ProjectCollaboratorQueryset.as_manager()

    project = models.ForeignKey(
        Project,
        on_delete=models.CASCADE,
        related_name="collaborators",
    )
    collaborator = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        limit_choices_to=models.Q(user_type__in=[User.TYPE_USER, User.TYPE_TEAM]),
    )
    role = models.CharField(max_length=10, choices=Roles.choices, default=Roles.READER)

    def __str__(self):
        return self.project.name + ": " + self.collaborator.username

    def clean(self) -> None:
        if self.project.owner == self.collaborator:
            raise ValidationError(_("Cannot add the project owner as a collaborator."))

        if self.project.owner.is_organization:
            organization = Organization.objects.get(pk=self.project.owner.pk)
            if self.collaborator.is_user:
                members_qs = organization.members.filter(member=self.collaborator)

                if members_qs.count() == 0:
                    raise ValidationError(
                        _(
                            "Cannot add a user who is not a member of the organization as a project collaborator."
                        )
                    )
            elif self.collaborator.is_team:
                team_qs = organization.teams.filter(pk=self.collaborator)
                if team_qs.count() == 0:

                    raise ValidationError(_("Team does not exist."))

        return super().clean()

    def save(self, *args, **kwargs):
        self.full_clean()
        return super().save(*args, **kwargs)


class ProjectRolesView(models.Model):
    user = models.ForeignKey(
        User,
        on_delete=models.DO_NOTHING,
        related_name="project_roles",
    )
    project = models.ForeignKey(
        "Project",
        on_delete=models.DO_NOTHING,
        related_name="user_roles",
    )
    name = models.CharField(max_length=100, choices=ProjectCollaborator.Roles.choices)
    origin = models.CharField(
        max_length=100, choices=ProjectQueryset.RoleOrigins.choices
    )
    is_valid = models.BooleanField()

    class Meta:
        db_table = "projects_with_roles_vw"
        managed = False


class Delta(models.Model):
    class Method(str, Enum):
        Create = "create"
        Delete = "delete"
        Patch = "patch"

    class Status(models.TextChoices):
        PENDING = "pending", _("Pending")
        STARTED = "started", _("Started")
        APPLIED = "applied", _("Applied")
        CONFLICT = "conflict", _("Conflict")
        NOT_APPLIED = "not_applied", _("Not_applied")
        ERROR = "error", _("Error")
        IGNORED = "ignored", _("Ignored")
        UNPERMITTED = "unpermitted", _("Unpermitted")

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    deltafile_id = models.UUIDField()
    project = models.ForeignKey(
        Project,
        on_delete=models.CASCADE,
        related_name="deltas",
    )
    content = JSONField()
    last_status = models.CharField(
        choices=Status.choices,
        default=Status.PENDING,
        max_length=32,
    )
    last_feedback = JSONField(null=True)
    last_modified_pk = models.TextField(null=True)
    last_apply_attempt_at = models.DateTimeField(null=True)
    last_apply_attempt_by = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        null=True,
    )
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    created_by = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        related_name="uploaded_deltas",
    )
    old_geom = models.GeometryField(null=True, srid=4326, dim=4)
    new_geom = models.GeometryField(null=True, srid=4326, dim=4)

    jobs_to_apply = models.ManyToManyField(
        to="ApplyJob",
        through="ApplyJobDelta",
    )

    def __str__(self):
        return str(self.id) + ", project: " + str(self.project.id)

    @staticmethod
    def get_status_summary(filters={}):
        rows = (
            Delta.objects.filter(**filters)
            .values("last_status")
            .annotate(count=Count("last_status"))
            .order_by()
        )

        rows_as_dict = {}
        for r in rows:
            rows_as_dict[r["last_status"]] = r["count"]

        counts = {}
        for status, _name in Delta.Status.choices:
            counts[status] = rows_as_dict.get(status, 0)

        return counts

    @property
    def short_id(self):
        return str(self.id)[0:8]

    @property
    def method(self):
        return self.content.get("method")

    @property
    def is_supported_regarding_owner_account(self):
        return (
            not self.project.has_online_vector_data
            or self.project.owner.useraccount.plan.is_external_db_supported
        )


class Job(models.Model):

    objects = InheritanceManager()

    class Type(models.TextChoices):
        PACKAGE = "package", _("Package")
        DELTA_APPLY = "delta_apply", _("Delta Apply")
        PROCESS_PROJECTFILE = "process_projectfile", _("Process QGIS Project File")

    class Status(models.TextChoices):
        PENDING = "pending", _("Pending")
        QUEUED = "queued", _("Queued")
        STARTED = "started", _("Started")
        FINISHED = "finished", _("Finished")
        STOPPED = "stopped", _("Stopped")
        FAILED = "failed", _("Failed")

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    project = models.ForeignKey(
        Project,
        on_delete=models.CASCADE,
        related_name="jobs",
    )
    type = models.CharField(max_length=32, choices=Type.choices)
    status = models.CharField(
        max_length=32, choices=Status.choices, default=Status.PENDING
    )
    output = models.TextField(null=True)
    feedback = JSONField(null=True)
    created_by = models.ForeignKey(User, on_delete=models.CASCADE)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    started_at = models.DateTimeField(blank=True, null=True, editable=False)
    finished_at = models.DateTimeField(blank=True, null=True, editable=False)

    @property
    def short_id(self) -> str:
        return str(self.id)[0:8]

    @property
    def fallback_output(self) -> str:
        # show whatever is the output if it is present
        if self.output:
            return ""

        if self.status == Job.Status.PENDING:
            return _(
                "The job is in pending status, it will be started as soon as there are available server resources."
            )
        elif self.status == Job.Status.QUEUED:
            return _(
                "The job is in queued status. Server resources are allocated and it will be started soon."
            )
        elif self.status == Job.Status.STARTED:
            return _("The job is in started status. Waiting for it to finish...")
        elif self.status == Job.Status.FINISHED:
            return _(
                "The job is in finished status. It finished successfully without any output."
            )
        elif self.status == Job.Status.STOPPED:
            return _("The job is in stopped status. Waiting to be continued...")
        elif self.status == Job.Status.FAILED:
            return _(
                "The job is in failed status. The execution failed due to server error. Please verify the project is configured properly and try again."
            )
        else:
            return _(
                "The job ended in unknown state. Please verify the project is configured properly, try again and contact QFieldCloud support for more information."
            )


class PackageJob(Job):
    def save(self, *args, **kwargs):
        self.type = self.Type.PACKAGE
        return super().save(*args, **kwargs)

    class Meta:
        verbose_name = "Job: package"
        verbose_name_plural = "Jobs: package"


class ProcessProjectfileJob(Job):
    def save(self, *args, **kwargs):
        self.type = self.Type.PROCESS_PROJECTFILE
        return super().save(*args, **kwargs)

    class Meta:
        verbose_name = "Job: process QGIS project file"
        verbose_name_plural = "Jobs: process QGIS project file"


class ApplyJob(Job):

    deltas_to_apply = models.ManyToManyField(
        to=Delta,
        through="ApplyJobDelta",
    )

    overwrite_conflicts = models.BooleanField(
        help_text=_(
            "If enabled, QFieldCloud will automatically overwrite conflicts while applying deltas."
        ),
    )

    def save(self, *args, **kwargs):
        self.type = self.Type.DELTA_APPLY
        return super().save(*args, **kwargs)

    class Meta:
        verbose_name = "Job: apply"
        verbose_name_plural = "Jobs: apply"


class ApplyJobDelta(models.Model):
    apply_job = models.ForeignKey(ApplyJob, on_delete=models.CASCADE)
    delta = models.ForeignKey(Delta, on_delete=models.CASCADE)
    status = models.CharField(
        choices=Delta.Status.choices, default=Delta.Status.PENDING, max_length=32
    )
    feedback = JSONField(null=True)
    modified_pk = models.TextField(null=True)

    def __str__(self):
        return f"{self.apply_job_id}:{self.delta_id}"


class Secret(models.Model):
    class Type(models.TextChoices):
        PGSERVICE = "pgservice", _("pg_service")
        ENVVAR = "envvar", _("Environment Variable")

    name = models.TextField(
        max_length=255,
        validators=[
            RegexValidator(
                r"^[A-Z]+[A-Z0-9_]+$",
                _(
                    "Must start with a capital letter and followed by capital letters, numbers or underscores."
                ),
            )
        ],
        help_text=_(
            _(
                "Must start with a capital letter and followed by capital letters, numbers or underscores."
            ),
        ),
    )
    type = models.CharField(max_length=32, choices=Type.choices)
    project = models.ForeignKey(
        Project, on_delete=models.CASCADE, related_name="secrets"
    )
    created_by = models.ForeignKey(
        User, on_delete=models.CASCADE, related_name="project_secrets"
    )
    created_at = models.DateTimeField(auto_now_add=True)
    value = django_cryptography.fields.encrypt(models.TextField())

    class Meta:
        ordering = ["project", "name"]
        constraints = [
            models.UniqueConstraint(
                fields=["project", "name"], name="secret_project_name_uniq"
            )
        ]


audited_models = [
    (User, {"exclude_fields": ["last_login", "updated_at"]}),
    (UserAccount, {}),
    (Organization, {}),
    (OrganizationMember, {}),
    (Team, {}),
    (TeamMember, {}),
    (
        Project,
        {
            "include_fields": [
                "id",
                "name",
                "description",
                "owner",
                "is_public",
                "owner",
                "created_at",
            ],
        },
    ),
    (ProjectCollaborator, {}),
    (
        Delta,
        {
            "include_fields": [
                "id",
                "deltafile_id",
                "project",
                "content",
                "last_status",
                "created_by",
            ],
        },
    ),
    (Secret, {"exclude_fields": ["value"]}),
]


def register_model_audits(subset: List[models.Model] = []) -> None:
    for model, kwargs in audited_models:
        if subset and model not in subset:
            continue
        auditlog.register(model, **kwargs)


def deregister_model_audits(subset: List[models.Model] = []) -> None:
    for model, _kwargs in audited_models:
        if subset and model not in subset:
            continue

        auditlog.unregister(model)


@contextlib.contextmanager
def no_audits(subset: List[models.Model] = []):
    try:
        deregister_model_audits(subset)
        yield
    finally:
        register_model_audits(subset)


# docker-app/qfieldcloud/core/permission_check.py

from typing import Any, Callable, List, Union

from django.http.request import HttpRequest
from django.http.response import HttpResponse, HttpResponseForbidden
from qfieldcloud.core import permissions_utils


def permission_check(
    perm: str, check_args: List[Union[str, Callable]] = []
) -> Callable:
    perm_check = getattr(permissions_utils, perm)

    def decorator_wrapper(func: Callable):
        def decorator(request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
            parsed_check_args = []

            for arg in check_args:
                if callable(arg):
                    parsed_check_args.append(arg(request))
                else:
                    parsed_check_args.append(getattr(request, arg))

            is_permitted = perm_check(request.user, *parsed_check_args)

            if is_permitted:
                return func(request, *args, **kwargs)
            else:
                return HttpResponseForbidden()

        return decorator

    return decorator_wrapper


# docker-app/qfieldcloud/core/permissions_utils.py

from typing import List, Literal, Union

from deprecated import deprecated
from django.utils.translation import gettext as _
from qfieldcloud.core.models import (
    Delta,
    Organization,
    OrganizationMember,
    OrganizationQueryset,
    Project,
    ProjectCollaborator,
    ProjectQueryset,
    Team,
)
from qfieldcloud.core.models import User as QfcUser


class CheckPermError(Exception):
    ...


class AlreadyCollaboratorError(CheckPermError):
    ...


class ReachedCollaboratorLimitError(CheckPermError):
    ...


class UserHasProjectRoleOrigins(CheckPermError):
    ...


class UserOrganizationRoleError(CheckPermError):
    ...


class ExpectedPremiumUserError(CheckPermError):
    ...


def _project_for_owner(user: QfcUser, project: Project):
    return Project.objects.for_user(user).filter(pk=project.pk)


def _organization_of_owner(user: QfcUser, organization: Organization):
    return Organization.objects.of_user(user).filter(pk=organization.pk)


def user_has_project_roles(
    user: QfcUser, project: Project, roles: List[ProjectCollaborator.Roles]
):
    return _project_for_owner(user, project).filter(user_role__in=roles).exists()


def check_user_has_project_role_origins(
    user: QfcUser, project: Project, origins: List[ProjectQueryset.RoleOrigins]
) -> Literal[True]:
    if _project_for_owner(user, project).filter(user_role_origin__in=origins).exists():
        return True

    raise UserHasProjectRoleOrigins(
        'User "{}" has not role origins "{}" on project {}.'.format(
            user.username,
            [origin.name for origin in origins],
            project.name,
        )
    )


def user_has_project_role_origins(
    user: QfcUser, project: Project, origins: List[ProjectQueryset.RoleOrigins]
) -> bool:
    try:
        return check_user_has_project_role_origins(user, project, origins)
    except CheckPermError:
        return False


def check_user_has_organization_roles(
    user: QfcUser, organization: Organization, roles: List[OrganizationMember.Roles]
) -> Literal[True]:
    if (
        _organization_of_owner(user, organization)
        .filter(membership_role__in=roles)
        .exists()
    ):
        return True

    raise UserOrganizationRoleError(
        _('User "{}" does not have {} roles in organization "{}"').format(
            user.username,
            [role.name for role in roles],
            organization.username,
        )
    )


def user_has_organization_roles(
    user: QfcUser, organization: Organization, roles: List[OrganizationMember.Roles]
) -> bool:
    try:
        return check_user_has_organization_roles(user, organization, roles)
    except CheckPermError:
        return False


def user_has_organization_role_origins(
    user: QfcUser,
    organization: Organization,
    origins: List[OrganizationQueryset.RoleOrigins],
):
    return (
        _organization_of_owner(user, organization)
        .filter(membership_role_origin__in=origins)
        .exists()
    )


def get_param_from_request(request, param):
    """Try to get the param from the request data or the request
    context, returns None otherwise"""

    result = request.data.get(param, None)
    if not result:
        result = request.parser_context["kwargs"].get(param, None)
    return result


def can_create_project(
    user: QfcUser, organization: Union[QfcUser, Organization] = None
) -> bool:
    """Return True if the `user` can create a project. Accepts additional
    `organizaiton` to check whether the user has permissions to do so on
    that organization. Return False otherwise."""

    if organization is None:
        return True
    if user == organization:
        return True

    if organization.is_organization:
        if not isinstance(organization, Organization):
            organization = organization.organization  # type: ignore
    else:
        return False

    if user_has_organization_roles(
        user, organization, [OrganizationMember.Roles.ADMIN]
    ):
        return True

    return False


def can_read_project(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
            ProjectCollaborator.Roles.REPORTER,
            ProjectCollaborator.Roles.READER,
        ],
    )


def can_update_project(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
        ],
    )


def can_delete_project(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
        ],
    )


def can_create_files(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
            ProjectCollaborator.Roles.REPORTER,
        ],
    )


def can_read_projects(user: QfcUser, _account: QfcUser) -> bool:
    return user.is_authenticated


def can_read_public_projects(user: QfcUser) -> bool:
    return user.is_authenticated


def can_read_files(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
            ProjectCollaborator.Roles.REPORTER,
            ProjectCollaborator.Roles.READER,
        ],
    )


def can_delete_files(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
        ],
    )


def can_create_deltas(user: QfcUser, project: Project) -> bool:
    """Whether the user can store deltas in a project."""
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
            ProjectCollaborator.Roles.REPORTER,
        ],
    )


def can_read_deltas(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
            ProjectCollaborator.Roles.REPORTER,
        ],
    )


def can_apply_pending_deltas_for_project(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
        ],
    )


@deprecated("Use `can_set_delta_status_for_project` instead")
def can_apply_deltas(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
            ProjectCollaborator.Roles.REPORTER,
        ],
    )


@deprecated("Use `can_set_delta_status_for_project` instead")
def can_overwrite_deltas(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
        ],
    )


def can_set_delta_status_for_project(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
        ],
    )


def can_set_delta_status(user: QfcUser, delta: Delta) -> bool:
    if not can_set_delta_status_for_project(user, delta.project):
        return False

    if delta.last_status not in (
        Delta.Status.PENDING,
        Delta.Status.CONFLICT,
        Delta.Status.NOT_APPLIED,
        Delta.Status.ERROR,
    ):
        return False

    return True


def can_create_delta(user: QfcUser, delta: Delta) -> bool:
    """Whether the user can store given delta."""
    project: Project = delta.project

    if user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
        ],
    ):
        return True

    if user_has_project_roles(user, project, [ProjectCollaborator.Roles.REPORTER]):
        if delta.method == Delta.Method.Create:
            return True

    return False


@deprecated("Use `can_set_delta_status` instead")
def can_retry_delta(user: QfcUser, delta: Delta) -> bool:
    if not can_apply_deltas(user, delta.project):
        return False

    if delta.last_status not in (
        Delta.Status.CONFLICT,
        Delta.Status.NOT_APPLIED,
        Delta.Status.ERROR,
    ):
        return False

    return True


@deprecated("Use `can_set_delta_status` instead")
def can_overwrite_delta(user: QfcUser, delta: Delta) -> bool:
    if not can_overwrite_deltas(user, delta.project):
        return False

    if delta.last_status not in (Delta.Status.CONFLICT):
        return False

    return True


@deprecated("Use `can_set_delta_status` instead")
def can_ignore_delta(user: QfcUser, delta: Delta) -> bool:
    if not can_apply_deltas(user, delta.project):
        return False

    if delta.last_status not in (
        Delta.Status.CONFLICT,
        Delta.Status.NOT_APPLIED,
        Delta.Status.ERROR,
    ):
        return False

    return True


def can_create_secrets(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
        ],
    )


def can_delete_secrets(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
        ],
    )


def can_list_users_organizations(user: QfcUser) -> bool:
    """Return True if the `user` can list users and organizations.
    Return False otherwise."""

    return True


def can_create_organizations(user: QfcUser) -> bool:
    return user.is_authenticated


def can_update_user(user: QfcUser, account: QfcUser) -> bool:
    if user == account:
        return True

    if user_has_organization_roles(user, account, [OrganizationMember.Roles.ADMIN]):
        return True

    return False


def can_delete_user(user: QfcUser, account: QfcUser) -> bool:
    if user == account:
        return True

    if user_has_organization_roles(user, account, [OrganizationMember.Roles.ADMIN]):
        return True

    return False


def can_create_collaborators(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
        ],
    )


def can_read_collaborators(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
        ],
    )


def can_update_collaborators(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
        ],
    )


def can_delete_collaborators(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
        ],
    )


def can_read_packages(user: QfcUser, project: Project) -> bool:
    return user_has_project_roles(
        user,
        project,
        [
            ProjectCollaborator.Roles.ADMIN,
            ProjectCollaborator.Roles.MANAGER,
            ProjectCollaborator.Roles.EDITOR,
            ProjectCollaborator.Roles.REPORTER,
            ProjectCollaborator.Roles.READER,
        ],
    )


def can_create_members(user: QfcUser, organization: Organization) -> bool:
    """Return True if the `user` can create members (incl. teams) of `organization`.
    Return False otherwise."""

    return user_has_organization_roles(
        user, organization, [OrganizationMember.Roles.ADMIN]
    )


def can_read_members(user: QfcUser, organization: Organization) -> bool:
    """Return True if the `user` can list members (incl. teams) of `organization`.
    Return False otherwise."""

    if not organization.is_organization:
        return False

    return True


def can_update_members(user: QfcUser, organization: Organization) -> bool:
    return user_has_organization_roles(
        user, organization, [OrganizationMember.Roles.ADMIN]
    )


def can_delete_members(user: QfcUser, organization: Organization) -> bool:
    return user_has_organization_roles(
        user, organization, [OrganizationMember.Roles.ADMIN]
    )


def check_can_become_collaborator(user: QfcUser, project: Project) -> bool:
    if user == project.owner:
        raise AlreadyCollaboratorError(
            _("Cannot add the project owner as a collaborator.")
        )

    if project.collaborators.filter(collaborator=user).count() > 0:
        raise AlreadyCollaboratorError(
            _('The user "{}" is already a collaborator of project "{}".').format(
                user.username, project.name
            )
        )

    max_premium_collaborators_per_private_project = (
        project.owner.useraccount.plan.max_premium_collaborators_per_private_project
    )
    if max_premium_collaborators_per_private_project >= 0 and not project.is_public:
        project_collaborators_count = project.direct_collaborators.count()
        if project_collaborators_count >= max_premium_collaborators_per_private_project:
            raise ReachedCollaboratorLimitError(
                _(
                    "The subscription plan of the project owner does not allow any additional collaborators. "
                    "Please remove some collaborators first."
                )
            )

    # Rules for organization projects
    if project.owner.is_organization:
        # And only members of these organizations can join
        check_user_has_organization_roles(
            user,
            project.owner,
            [OrganizationMember.Roles.MEMBER, OrganizationMember.Roles.ADMIN],
        )

    # Rules for private projects
    if not project.is_public:
        if not user.useraccount.plan.is_premium:
            raise ExpectedPremiumUserError(
                _('User "{}" must be a premium user.').format(user.username)
            )

    return True


def can_become_collaborator(user: QfcUser, project: Project) -> bool:
    try:
        return check_can_become_collaborator()
    except CheckPermError:
        return False


def can_read_geodb(user: QfcUser, profile: QfcUser) -> bool:
    if not hasattr(profile, "useraccount") or not profile.useraccount.is_geodb_enabled:
        return False

    if can_update_user(user, profile):
        return True

    return False


def can_create_geodb(user: QfcUser, profile: QfcUser) -> bool:
    if not hasattr(profile, "useraccount") or not profile.useraccount.is_geodb_enabled:
        return False

    if profile.has_geodb:
        return False

    if can_update_user(user, profile):
        return True

    return False


def can_delete_geodb(user: QfcUser, profile: QfcUser) -> bool:
    if not hasattr(profile, "useraccount") or not profile.useraccount.is_geodb_enabled:
        return False

    if not profile.has_geodb:
        return False

    if can_update_user(user, profile):
        return True

    return False


def can_become_member(user: QfcUser, organization: Organization) -> bool:
    if user.user_type == QfcUser.TYPE_ORGANIZATION:
        return False

    if user.user_type == QfcUser.TYPE_TEAM:
        return Team.objects.get(pk=user.pk).team_organization == organization

    return not user_has_organization_role_origins(
        user,
        organization,
        [
            OrganizationQueryset.RoleOrigins.ORGANIZATIONOWNER,
            OrganizationQueryset.RoleOrigins.ORGANIZATIONMEMBER,
        ],
    )


def can_send_invitations(user: QfcUser, account: QfcUser) -> bool:
    if user.pk != account.pk:
        return False

    if account.is_user:
        return True

    return False


# docker-app/qfieldcloud/core/querysets_utils.py

from functools import reduce
from operator import and_, or_

from django.db.models import Q
from django.db.models.manager import BaseManager
from qfieldcloud.core.models import (
    Delta,
    Organization,
    OrganizationMember,
    Project,
    ProjectCollaborator,
    Team,
    TeamMember,
    User,
)


def get_organization_teams(organization):
    return Team.objects.filter(team_organization=organization)


def get_team_members(team):
    return TeamMember.objects.filter(team=team)


def get_organization_members(organization):
    return OrganizationMember.objects.filter(organization=organization)


def get_project_deltas(project):
    """Return a queryset of all available deltas of a specific project."""
    return Delta.objects.filter(project=project)


def get_users(
    username: str,
    project: Project = None,
    organization: Organization = None,
    exclude_organizations: bool = False,
    exclude_teams: bool = False,
    invert: bool = False,
) -> BaseManager:
    assert (
        project is None or organization is None
    ), "Cannot have the project and organization filters set simultaneously"

    if username:
        users = User.objects.filter(
            Q(username__icontains=username) | Q(email__icontains=username)
        )
    else:
        users = User.objects.all()

    if exclude_organizations:
        users = users.exclude(user_type=User.TYPE_ORGANIZATION)

    if exclude_teams:
        users = users.exclude(user_type=User.TYPE_TEAM)
    else:
        if project:
            users = users.filter(
                ~Q(user_type=User.TYPE_TEAM)
                | (
                    Q(user_type=User.TYPE_TEAM)
                    & Q(pk__in=Team.objects.filter(team_organization=project.owner))
                )
            )

    # one day conditions can be more than just pk check, please keep it for now
    conditions = []
    # exclude the already existing collaborators and the project owner
    if project:
        collaborator_ids = ProjectCollaborator.objects.filter(
            project=project
        ).values_list("collaborator", flat=True)
        user_ids = [*collaborator_ids, project.owner.pk]
        conditions = [Q(pk__in=user_ids)]

    # exclude the already existing members, the organization owner and the organization itself from the returned users
    elif organization:
        # exclude all teams that are not of the current organization
        users = users.filter(
            ~Q(user_type=User.TYPE_TEAM)
            | Q(pk__in=Team.objects.filter(team_organization=organization))
        )

        member_ids = OrganizationMember.objects.filter(
            organization=organization
        ).values_list("member", flat=True)
        user_ids = [
            *member_ids,
            organization.organization_owner.pk,
            organization.user_ptr.pk,
        ]
        conditions = [Q(pk__in=user_ids)]

    if conditions:
        if invert:
            users = users.filter(reduce(and_, [c for c in conditions]))
        else:
            users = users.exclude(reduce(or_, [c for c in conditions]))

    return users.order_by("-username")


# docker-app/qfieldcloud/core/rest_utils.py

import logging

from django.conf import settings
from django.core import exceptions
from qfieldcloud.core import exceptions as qfieldcloud_exceptions
from rest_framework import exceptions as rest_exceptions
from rest_framework.response import Response

logger = logging.getLogger(__name__)


def exception_handler(exc, context):

    if isinstance(exc, qfieldcloud_exceptions.QFieldCloudException):
        pass
    elif isinstance(exc, rest_exceptions.AuthenticationFailed):
        exc = qfieldcloud_exceptions.AuthenticationFailedError()
    elif isinstance(exc, rest_exceptions.NotAuthenticated):
        exc = qfieldcloud_exceptions.NotAuthenticatedError()
    elif isinstance(exc, rest_exceptions.APIException):
        exc = qfieldcloud_exceptions.APIError(
            status_code=exc.status_code, detail=exc.detail
        )
    elif isinstance(exc, exceptions.ObjectDoesNotExist):
        exc = qfieldcloud_exceptions.ObjectNotFoundError(detail=str(exc))
    elif isinstance(exc, exceptions.ValidationError):
        exc = qfieldcloud_exceptions.ValidationError(detail=str(exc))
    else:
        # When running tests, we rethrow the exception, so we get a full trace to
        # help with debugging
        if settings.IN_TEST_SUITE:
            raise exc
        logging.exception(exc)
        exc = qfieldcloud_exceptions.QFieldCloudException(detail=str(exc))

    body = {
        "code": exc.code,
        "message": exc.message,
    }

    if settings.DEBUG:
        body["debug"] = {
            "view": str(context["view"]),
            "args": context["args"],
            "kwargs": context["kwargs"],
            "request": str(context["request"]),
            "detail": exc.detail,
        }

    logging.exception(exc)

    return Response(
        body,
        status=exc.status_code,
    )


# docker-app/qfieldcloud/core/serializers.py

import os
from typing import Optional

from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core import exceptions
from qfieldcloud.core.models import (
    ApplyJob,
    Delta,
    Job,
    Organization,
    OrganizationMember,
    PackageJob,
    ProcessProjectfileJob,
    Project,
    ProjectCollaborator,
    Team,
)
from rest_framework import serializers
from rest_framework.exceptions import ValidationError

User = get_user_model()


def get_avatar_url(user: User) -> Optional[str]:
    if hasattr(user, "useraccount") and user.useraccount.avatar_url:
        site = Site.objects.get_current()
        port = os.environ.get("WEB_HTTPS_PORT")
        port = f":{port}" if port != "443" else ""
        return f"https://{site.domain}{port}{user.useraccount.avatar_url}"
    return None


class UserSerializer:
    class Meta:
        model = User
        fields = ("username",)


class ProjectSerializer(serializers.ModelSerializer):
    owner = serializers.StringRelatedField()
    user_role = serializers.CharField(read_only=True)
    user_role_origin = serializers.CharField(read_only=True)
    private = serializers.BooleanField(allow_null=True, default=None)

    def to_internal_value(self, data):
        internal_data = super().to_internal_value(data)
        owner_username = data.get("owner")

        if owner_username:
            try:
                internal_data["owner"] = User.objects.get(username=owner_username)
            except User.DoesNotExist:
                raise ValidationError(
                    {"owner": ["Invalid owner username"]},
                    code="invalid",
                )
        else:
            internal_data["owner"] = self.context["request"].user

        if "private" in internal_data:
            if internal_data["private"] is not None:
                internal_data["is_public"] = not internal_data["private"]

            del internal_data["private"]

        return internal_data

    class Meta:
        fields = (
            "id",
            "name",
            "owner",
            "description",
            # remove "private" field one day
            "private",
            "is_public",
            "created_at",
            "updated_at",
            "data_last_packaged_at",
            "data_last_updated_at",
            "can_repackage",
            "needs_repackaging",
            "status",
            "user_role",
            "user_role_origin",
        )
        model = Project


class CompleteUserSerializer(serializers.ModelSerializer):
    avatar_url = serializers.SerializerMethodField()

    def get_avatar_url(self, obj):
        return get_avatar_url(obj)

    class Meta:
        model = User
        fields = (
            "username",
            "user_type",
            "full_name",
            "email",
            "avatar_url",
            "first_name",
            "last_name",
        )
        read_only_fields = ("full_name", "avatar_url")


class PublicInfoUserSerializer(serializers.ModelSerializer):
    avatar_url = serializers.SerializerMethodField()
    username_display = serializers.SerializerMethodField()

    def get_avatar_url(self, obj):
        return get_avatar_url(obj)

    def get_username_display(self, obj):
        if obj.user_type == obj.TYPE_TEAM:
            team = Team.objects.get(id=obj.id)
            return team.username.replace(f"@{team.team_organization.username}/", "")
        else:
            return obj.username

    class Meta:
        model = User
        fields = (
            "username",
            "user_type",
            "full_name",
            "avatar_url",
            "username_display",
        )
        read_only_fields = ("full_name", "avatar_url", "username_display")


class OrganizationSerializer(serializers.ModelSerializer):
    organization_owner = serializers.StringRelatedField()
    members = serializers.SerializerMethodField()
    avatar_url = serializers.SerializerMethodField()
    membership_role = serializers.CharField(read_only=True)
    membership_role_origin = serializers.CharField(read_only=True)
    membership_is_public = serializers.BooleanField(
        read_only=True, source="membership_role_is_public"
    )

    def get_members(self, obj):
        return [
            m["member__username"]
            for m in OrganizationMember.objects.filter(
                organization=obj, is_public=True
            ).values("member__username")
        ]

    def get_avatar_url(self, obj):
        return get_avatar_url(obj)

    class Meta:
        model = Organization
        fields = (
            "username",
            "user_type",
            "email",
            "avatar_url",
            "members",
            "organization_owner",
            "membership_role",
            "membership_role_origin",
            "membership_is_public",
        )


class ProjectCollaboratorSerializer(serializers.ModelSerializer):
    collaborator = serializers.StringRelatedField()
    role = serializers.CharField()

    class Meta:
        model = ProjectCollaborator
        fields = ("collaborator", "role")


class OrganizationMemberSerializer(serializers.ModelSerializer):
    member = serializers.StringRelatedField()
    role = serializers.CharField()

    class Meta:
        model = OrganizationMember
        fields = ("member", "role")


class TokenSerializer(serializers.ModelSerializer):
    username = serializers.CharField(source="user.username")
    expires_at = serializers.DateTimeField()
    user_type = serializers.CharField(source="user.user_type")
    first_name = serializers.CharField(source="user.first_name")
    last_name = serializers.CharField(source="user.last_name")
    full_name = serializers.CharField(source="user.full_name")
    token = serializers.CharField(source="key")
    email = serializers.SerializerMethodField()
    avatar_url = serializers.SerializerMethodField()

    def get_email(self, obj):
        return obj.user.email

    def get_avatar_url(self, obj):
        return get_avatar_url(obj.user)

    class Meta:
        model = AuthToken
        fields = (
            "token",
            "expires_at",
            "username",
            "user_type",
            "email",
            "avatar_url",
            "first_name",
            "last_name",
            "full_name",
        )
        read_only_fields = (
            "token",
            "expires_at",
            "full_name",
            "avatar_url",
        )


class StatusChoiceField(serializers.ChoiceField):
    def to_representation(self, obj):
        return self._choices[obj]

    def to_internal_value(self, data):
        for i in self._choices:
            if self._choices[i] == data:
                return i
        raise serializers.ValidationError(
            "Invalid status. Acceptable values are {0}.".format(
                list(self._choices.values())
            )
        )


class DeltaSerializer(serializers.ModelSerializer):
    created_by = serializers.StringRelatedField()
    status = serializers.SerializerMethodField()
    output = serializers.CharField(source="last_feedback")

    def get_status(self, obj):
        if obj.last_status == Delta.Status.PENDING:
            return "STATUS_PENDING"
        elif obj.last_status == Delta.Status.STARTED:
            return "STATUS_BUSY"
        elif obj.last_status == Delta.Status.APPLIED:
            return "STATUS_APPLIED"
        elif obj.last_status == Delta.Status.CONFLICT:
            return "STATUS_CONFLICT"
        elif obj.last_status == Delta.Status.NOT_APPLIED:
            return "STATUS_NOT_APPLIED"
        elif obj.last_status == Delta.Status.ERROR:
            return "STATUS_ERROR"
        elif obj.last_status == Delta.Status.IGNORED:
            return "STATUS_IGNORED"
        elif obj.last_status == Delta.Status.UNPERMITTED:
            return "STATUS_UNPERMITTED"
        else:
            return "STATUS_ERROR"

    class Meta:
        model = Delta
        fields = (
            "id",
            "deltafile_id",
            "created_by",
            "created_at",
            "updated_at",
            "status",
            "output",
            "last_status",
            "last_feedback",
            "content",
        )


class ExportJobSerializer(serializers.ModelSerializer):
    # TODO layers used to hold information about layer validity. No longer needed.
    layers = serializers.SerializerMethodField()
    status = serializers.SerializerMethodField(initial="STATUS_ERROR")

    def get_initial(self):
        if self.instance is None:
            return {
                "layers": None,
                "output": "",
                "status": "STATUS_ERROR",
            }
        return super().get_initial()

    def get_layers(self, obj):
        if not obj.feedback:
            return None

        if obj.status != Job.Status.FINISHED:
            return None

        if obj.feedback.get("feedback_version") == "2.0":
            return obj.feedback["outputs"]["qgis_layers_data"]["layers_by_id"]
        else:
            steps = obj.feedback.get("steps", [])
            if len(steps) > 2 and steps[1].get("stage", 1) == 2:
                return steps[1]["outputs"]["layer_checks"]

        return None

    def get_status(self, obj):
        if obj.status == Job.Status.PENDING:
            return "STATUS_PENDING"
        elif obj.status == Job.Status.QUEUED:
            return "STATUS_BUSY"
        elif obj.status == Job.Status.STARTED:
            return "STATUS_BUSY"
        elif obj.status == Job.Status.STOPPED:
            return "STATUS_BUSY"
        elif obj.status == Job.Status.FINISHED:
            return "STATUS_EXPORTED"
        elif obj.status == Job.Status.FAILED:
            return "STATUS_ERROR"
        else:
            return "STATUS_ERROR"

    class Meta:
        model = PackageJob
        fields = ("status", "layers", "output")


class JobMixin:
    project_id = serializers.PrimaryKeyRelatedField(queryset=Project.objects.all())

    def to_internal_value(self, data):
        internal_data = super().to_internal_value(data)
        internal_data["created_by"] = self.context["request"].user
        internal_data["project"] = Project.objects.get(pk=data.get("project_id"))

        return internal_data

    def get_lastest_not_finished_job(self) -> Optional[Job]:
        ModelClass: Job = self.Meta.model
        last_active_job = (
            ModelClass.objects.filter(
                project=self.initial_data.get("project_id"),
                status__in=[Job.Status.PENDING, Job.Status.QUEUED, Job.Status.STARTED],
            )
            .only("id")
            .order_by("-started_at", "-created_at")
            .last()
        )

        return last_active_job

    class Meta:
        model = PackageJob
        fields = (
            "id",
            "created_at",
            "created_by",
            "finished_at",
            "project_id",
            "started_at",
            "status",
            "type",
            "updated_at",
            "feedback",
            "output",
        )

        read_only_fields = (
            "id",
            "created_at",
            "created_by",
            "finished_at",
            "started_at",
            "status",
            "updated_at",
            "feedback",
            "output",
        )


class PackageJobSerializer(JobMixin, serializers.ModelSerializer):
    def get_lastest_not_finished_job(self) -> Optional[Job]:
        job = super().get_lastest_not_finished_job()
        if job:
            return job

        internal_value = self.to_internal_value(self.initial_data)

        if not internal_value["project"].project_filename:
            raise exceptions.NoQGISProjectError()

    class Meta(JobMixin.Meta):
        model = PackageJob
        allow_parallel_jobs = False


class ApplyJobSerializer(JobMixin, serializers.ModelSerializer):
    class Meta(JobMixin.Meta):
        model = ApplyJob
        allow_parallel_jobs = True


class ProcessProjectfileJobSerializer(JobMixin, serializers.ModelSerializer):
    class Meta(JobMixin.Meta):
        model = ProcessProjectfileJob
        allow_parallel_jobs = True


class JobSerializer(serializers.ModelSerializer):
    def get_lastest_not_finished_job(self):
        return None

    def get_fields(self, *args, **kwargs):
        fields = super().get_fields(*args, **kwargs)
        request = self.context.get("request")

        if request and "job_id" not in request.parser_context.get("kwargs", {}):
            fields.pop("output", None)
            fields.pop("feedback", None)
            fields.pop("layers", None)

        return fields

    class Meta:
        model = Job
        fields = (
            "id",
            "created_at",
            "created_by",
            "finished_at",
            "project_id",
            "started_at",
            "status",
            "type",
            "updated_at",
            "feedback",
            "output",
        )
        read_only_fields = (
            "id",
            "created_at",
            "created_by",
            "finished_at",
            "started_at",
            "status",
            "updated_at",
            "feedback",
            "output",
        )
        order_by = "-created_at"
        allow_parallel_jobs = True


# docker-app/qfieldcloud/core/signals.py

from axes.signals import user_locked_out
from django.dispatch import receiver
from django.utils.translation import gettext as _
from rest_framework.exceptions import PermissionDenied


@receiver(user_locked_out)
def raise_permission_denied(*args, **kwargs):
    raise PermissionDenied(_("Too many failed login attempts"))


# docker-app/qfieldcloud/core/sql_config.py

from migrate_sql.config import SQLItem

sql_items = [
    SQLItem(
        "project_user_collaborators_vw",
        r"""
            CREATE OR REPLACE VIEW project_user_collaborators_vw AS

            SELECT
                C1.*,
                (
                    O1.user_ptr_id IS NULL AND P1.is_public
                    OR OM1.id IS NOT NULL
                ) AS "is_valid"
            FROM
                core_projectcollaborator C1
                INNER JOIN core_project P1 ON P1.id = C1.project_id
                LEFT JOIN core_organization O1 ON O1.user_ptr_id = P1.owner_id
                LEFT JOIN core_organizationmember OM1 ON OM1.organization_id = O1.user_ptr_id
        """,
        r"""
            DROP VIEW project_user_collaborators_vw;
        """,
    ),
    SQLItem(
        "projects_with_roles_vw_seq",
        r"""
            CREATE SEQUENCE IF NOT EXISTS projects_with_roles_vw_seq CACHE 5000 CYCLE
        """,
        r"""
            DROP SEQUENCE IF EXISTS projects_with_roles_vw_seq
        """,
    ),
    SQLItem(
        "projects_with_roles_vw",
        r"""
            CREATE OR REPLACE VIEW projects_with_roles_vw AS

            WITH project_owner AS (
                SELECT
                    1 AS rank,
                    P1."id" AS "project_id",
                    P1."owner_id" AS "user_id",
                    'admin' AS "name",
                    'project_owner' AS "origin",
                    TRUE AS "is_valid"
                FROM
                    "core_project" P1
                    INNER JOIN "core_user" U1 ON (P1."owner_id" = U1."id")
                WHERE
                    U1."user_type" = 1
            ),
            organization_owner AS (
                SELECT
                    2 AS rank,
                    P1."id" AS "project_id",
                    O1."organization_owner_id" AS "user_id",
                    'admin' AS "name",
                    'organization_owner' AS "origin",
                    TRUE AS "is_valid"
                FROM
                    "core_organization" O1
                    INNER JOIN "core_project" P1 ON (P1."owner_id" = O1."user_ptr_id")
            ),
            organization_admin AS (
                SELECT
                    3 AS rank,
                    P1."id" AS "project_id",
                    OM1."member_id" AS "user_id",
                    'admin' AS "name",
                    'organization_admin' AS "origin",
                    TRUE AS "is_valid"
                FROM
                    "core_organizationmember" OM1
                    INNER JOIN "core_project" P1 ON (P1."owner_id" = OM1."organization_id")
                WHERE
                    (
                        OM1."role" = 'admin'
                    )
            ),
            project_collaborator AS (
                SELECT
                    4 AS rank,
                    C1."project_id",
                    C1."collaborator_id" AS "user_id",
                    C1."role" AS "name",
                    'collaborator' AS "origin",
                    P1.is_public OR U1.user_type = 2 AS "is_valid"
                FROM
                    "project_user_collaborators_vw" C1
                    INNER JOIN "core_project" P1 ON (P1."id" = C1."project_id")
                    INNER JOIN "core_user" U1 ON (P1."owner_id" = U1."id")
                WHERE
                    C1."is_valid" = TRUE
            ),
            project_collaborator_team AS (
                SELECT
                    5 AS rank,
                    C1."project_id",
                    TM1."member_id" AS "user_id",
                    C1."role" AS "name",
                    'team_member' AS "origin",
                    TRUE AS "is_valid"
                FROM
                    "core_projectcollaborator" C1
                    INNER JOIN "core_user" U1 ON (C1."collaborator_id" = U1."id")
                    INNER JOIN "core_team" T1 ON (U1."id" = T1."user_ptr_id")
                    INNER JOIN "core_teammember" TM1 ON (T1."user_ptr_id" = TM1."team_id")
                    INNER JOIN "core_project" P1 ON (P1."id" = C1."project_id")
            ),
            public_project AS (
                SELECT
                    6 AS rank,
                    P1."id" AS "project_id",
                    U1."id" AS "user_id",
                    'reader' AS "name",
                    'public' AS "origin",
                    TRUE AS "is_valid"
                FROM
                    "core_project" P1
                    CROSS JOIN "core_user" U1
                WHERE
                    is_public = TRUE
            )
            SELECT DISTINCT ON(project_id, user_id)
                nextval('projects_with_roles_vw_seq') id,
                R1.*
            FROM (
                SELECT * FROM project_owner
                UNION
                SELECT * FROM organization_owner
                UNION
                SELECT * FROM organization_admin
                UNION
                SELECT * FROM project_collaborator
                UNION
                SELECT * FROM project_collaborator_team
                UNION
                SELECT * FROM public_project
            ) R1
            ORDER BY project_id, user_id, rank
        """,
        r"""
            DROP VIEW projects_with_roles_vw;
        """,
    ),
    SQLItem(
        "organizations_with_roles_vw_seq",
        r"""
            CREATE SEQUENCE IF NOT EXISTS organizations_with_roles_vw_seq CACHE 5000 CYCLE
        """,
        r"""
            DROP SEQUENCE IF EXISTS organizations_with_roles_vw_seq
        """,
    ),
    SQLItem(
        "organizations_with_roles_vw",
        r"""
            CREATE OR REPLACE VIEW organizations_with_roles_vw AS
            WITH organization_owner AS (
                SELECT
                    1 AS rank,
                    ORG1."user_ptr_id" AS "organization_id",
                    ORG1."organization_owner_id" AS "user_id",
                    'admin' AS "name",
                    'organization_owner' AS "origin",
                    TRUE AS "is_public"
                FROM
                    "core_organization" ORG1
            ),
            organization_member AS (
                SELECT
                    2 AS rank,
                    M1."organization_id" AS "organization_id",
                    M1."member_id" AS "user_id",
                    M1.role AS "name",
                    'organization_member' AS "origin",
                    M1."is_public" AS "is_public"
                FROM
                    "core_organizationmember" M1
            )
            SELECT DISTINCT ON(organization_id, user_id)
                nextval('organizations_with_roles_vw_seq') id,
                R1.*
            FROM (
                SELECT * FROM organization_owner
                UNION
                SELECT * FROM organization_member
            ) R1
            ORDER BY organization_id, user_id, rank
        """,
        r"""
            DROP VIEW IF EXISTS "organizations_with_roles_vw";
        """,
    ),
    SQLItem(
        "core_delta_geom_trigger_func",
        r"""
            CREATE OR REPLACE FUNCTION core_delta_geom_trigger_func()
            RETURNS trigger
            AS
            $$
                DECLARE
                    srid int;
                BEGIN
                    SELECT CASE
                        WHEN jsonb_extract_path_text(NEW.content, 'localLayerCrs') ~ '^EPSG:\d{1,10}$'
                        THEN
                            REGEXP_REPLACE(jsonb_extract_path_text(NEW.content, 'localLayerCrs'), '\D*', '', 'g')::int
                        ELSE
                            NULL
                        END INTO srid;
                    NEW.old_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'old', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );
                    NEW.new_geom := ST_Transform( ST_SetSRID( ST_Force2D( ST_GeomFromText( REPLACE( jsonb_extract_path_text(NEW.content, 'new', 'geometry'), 'nan', '0' ) ) ), srid ), 4326 );

                    IF ST_GeometryType(NEW.old_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')
                    THEN
                        NEW.old_geom := ST_CurveToLine(NEW.old_geom);
                    END IF;

                    IF ST_GeometryType(NEW.new_geom) IN ('ST_CircularString', 'ST_CompoundCurve', 'ST_CurvePolygon', 'ST_MultiCurve', 'ST_MultiSurface')
                    THEN
                        NEW.new_geom := ST_CurveToLine(NEW.new_geom);
                    END IF;

                    RETURN NEW;
                END;
            $$
            LANGUAGE PLPGSQL
        """,
        r"""
            DROP FUNCTION IF EXISTS core_delta_geom_trigger_func()
        """,
    ),
    SQLItem(
        "core_delta_geom_update_trigger",
        r"""
            CREATE TRIGGER core_delta_geom_update_trigger BEFORE UPDATE ON core_delta
            FOR EACH ROW
            WHEN (OLD.content IS DISTINCT FROM NEW.content)
            EXECUTE FUNCTION core_delta_geom_trigger_func()
        """,
        r"""
            DROP TRIGGER IF EXISTS core_delta_geom_update_trigger ON core_delta
        """,
    ),
    SQLItem(
        "core_delta_geom_insert_trigger",
        r"""
            CREATE TRIGGER core_delta_geom_insert_trigger BEFORE INSERT ON core_delta
            FOR EACH ROW
            EXECUTE FUNCTION core_delta_geom_trigger_func()
        """,
        r"""
            DROP TRIGGER IF EXISTS core_delta_geom_insert_trigger ON core_delta
        """,
    ),
]


# docker-app/qfieldcloud/core/urls.py

from django.urls import include, path
from qfieldcloud.core.views import (
    collaborators_views,
    deltas_views,
    files_views,
    jobs_views,
    members_views,
    package_views,
    projects_views,
    qfield_files_views,
    status_views,
    users_views,
)
from rest_framework.routers import DefaultRouter

router = DefaultRouter()
router.register(r"projects", projects_views.ProjectViewSet, basename="project")
router.register(r"jobs", jobs_views.JobViewSet, basename="jobs")

"""
TODO future URL refactor
projects/
projects/
projects/<uuid:project_id>/
projects/<uuid:project_id>/files/
projects/<uuid:project_id>/files/<path:filename>/
projects/<uuid:project_id>/jobs/
projects/<uuid:project_id>/jobs/<uuid:job_id>/
projects/<uuid:project_id>/packages/
projects/<uuid:project_id>/packages/latest/files/
projects/<uuid:project_id>/packages/latest/files/<path:filename>/
projects/<uuid:project_id>/deltas/
projects/<uuid:project_id>/deltas/<uuid:delta_id>/
projects/<uuid:project_id>/collaborators/
organizations/
organizations/<str:organization_name>/
organizations/<str:organization_name>/members/
organizations/<str:organization_name>/teams/
organizations/<str:organization_name>/teams/<str:team_name>/members/
"""

urlpatterns = [
    path("projects/public/", projects_views.PublicProjectsListView.as_view()),
    path("", include(router.urls)),
    path("users/", users_views.ListUsersView.as_view()),
    path(
        "users/<str:username>/organizations/",
        users_views.ListUserOrganizationsView.as_view(),
    ),
    path("users/<str:username>/", users_views.RetrieveUpdateUserView.as_view()),
    path(
        "collaborators/<uuid:projectid>/",
        collaborators_views.ListCreateCollaboratorsView.as_view(),
    ),
    path(
        "collaborators/<uuid:projectid>/<str:username>/",
        collaborators_views.GetUpdateDestroyCollaboratorView.as_view(),
    ),
    path("files/<uuid:projectid>/", files_views.ListFilesView.as_view()),
    path(
        "files/<uuid:projectid>/<path:filename>/",
        files_views.DownloadPushDeleteFileView.as_view(),
        name="project_file_download",
    ),
    path(
        "files/meta/<uuid:projectid>/<path:filename>",
        files_views.ProjectMetafilesView.as_view(),
        name="project_metafiles",
    ),
    path(
        "files/public/<path:filename>",
        files_views.PublicFilesView.as_view(),
        name="public_files",
    ),
    path(
        "packages/<uuid:project_id>/latest/",
        package_views.LatestPackageView.as_view(),
    ),
    path(
        "packages/<uuid:project_id>/latest/files/<path:filename>/",
        package_views.LatestPackageDownloadFilesView.as_view(),
    ),
    path(
        "packages/<uuid:project_id>/<uuid:job_id>/files/<path:filename>/",
        package_views.PackageUploadFilesView.as_view(),
    ),
    path("qfield-files/<uuid:projectid>/", qfield_files_views.ListFilesView.as_view()),
    path(
        "qfield-files/<uuid:projectid>/<path:filename>/",
        qfield_files_views.DownloadFileView.as_view(),
    ),
    path(
        "qfield-files/export/<uuid:projectid>/",
        qfield_files_views.PackageView.as_view(),
    ),
    path("members/<str:organization>/", members_views.ListCreateMembersView.as_view()),
    path(
        "members/<str:organization>/<str:username>/",
        members_views.GetUpdateDestroyMemberView.as_view(),
    ),
    path("status/", status_views.APIStatusView.as_view()),
    path("deltas/<uuid:projectid>/", deltas_views.ListCreateDeltasView.as_view()),
    path(
        "deltas/<uuid:projectid>/<uuid:deltafileid>/",
        deltas_views.ListDeltasByDeltafileView.as_view(),
    ),
    path("deltas/apply/<uuid:projectid>/", deltas_views.ApplyView.as_view()),
]


# docker-app/qfieldcloud/core/utils.py

import hashlib
import io
import json
import logging
import os
import posixpath
from datetime import datetime
from pathlib import PurePath
from typing import IO, Iterable, List, NamedTuple, Optional, Union

import boto3
import jsonschema
import mypy_boto3_s3
from botocore.errorfactory import ClientError
from django.conf import settings
from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
from redis import Redis, exceptions

logger = logging.getLogger(__name__)


class S3PrefixPath(NamedTuple):
    Key: str


class S3Object(NamedTuple):
    name: str
    key: str
    last_modified: datetime
    size: int
    etag: str
    md5sum: str


class S3ObjectVersion:
    def __init__(
        self, name: str, data: mypy_boto3_s3.service_resource.ObjectVersion
    ) -> None:
        self.name = name
        self._data = data

    @property
    def id(self) -> str:
        """Returns the version id"""
        # NOTE id and version_id are the same thing
        return self._data.id

    @property
    def key(self) -> str:
        return self._data.key

    @property
    def last_modified(self) -> datetime:
        return self._data.last_modified

    @property
    def size(self) -> int:
        return self._data.size

    @property
    def e_tag(self) -> str:
        return self._data.e_tag

    @property
    def md5sum(self) -> str:
        return self._data.e_tag.replace('"', "")

    @property
    def is_latest(self) -> bool:
        return self._data.is_latest

    @property
    def display(self) -> str:
        return self.last_modified.strftime("v%Y%m%d%H%M%S")


class S3ObjectWithVersions(NamedTuple):
    latest: S3ObjectVersion
    versions: List[S3ObjectVersion]

    @property
    def total_size(self) -> int:
        """Total size of all versions"""
        # latest is also in versions
        return sum(v.size for v in self.versions)

    def delete(self):
        bucket = get_s3_bucket()

        return bucket.object_versions.filter(Prefix=self.latest.key).delete()


def redis_is_running() -> bool:
    try:
        connection = Redis(
            "redis", password=os.environ.get("REDIS_PASSWORD"), port=6379
        )
        connection.set("foo", "bar")
    except exceptions.ConnectionError:
        return False

    return True


def get_s3_session() -> boto3.Session:
    """Get a new S3 Session instance using Django settings"""

    session = boto3.Session(
        aws_access_key_id=settings.STORAGE_ACCESS_KEY_ID,
        aws_secret_access_key=settings.STORAGE_SECRET_ACCESS_KEY,
        region_name=settings.STORAGE_REGION_NAME,
    )
    return session


def get_s3_bucket() -> mypy_boto3_s3.service_resource.Bucket:
    """Get a new S3 Bucket instance using Django settings"""

    session = get_s3_session()

    # Get the bucket objects
    s3 = session.resource("s3", endpoint_url=settings.STORAGE_ENDPOINT_URL)
    return s3.Bucket(settings.STORAGE_BUCKET_NAME)


def get_s3_client() -> mypy_boto3_s3.Client:
    """Get a new S3 client instance using Django settings"""

    s3_client = boto3.client(
        "s3",
        region_name=settings.STORAGE_REGION_NAME,
        aws_access_key_id=settings.STORAGE_ACCESS_KEY_ID,
        aws_secret_access_key=settings.STORAGE_SECRET_ACCESS_KEY,
        endpoint_url=settings.STORAGE_ENDPOINT_URL,
    )
    return s3_client


def get_sha256(file: IO) -> str:
    """Return the sha256 hash of the file"""
    if type(file) is InMemoryUploadedFile or type(file) is TemporaryUploadedFile:
        return _get_sha256_memory_file(file)
    else:
        return _get_sha256_file(file)


def _get_sha256_memory_file(
    file: Union[InMemoryUploadedFile, TemporaryUploadedFile]
) -> str:
    BLOCKSIZE = 65536
    hasher = hashlib.sha256()

    for chunk in file.chunks(BLOCKSIZE):
        hasher.update(chunk)

    file.seek(0)
    return hasher.hexdigest()


def _get_sha256_file(file: IO) -> str:
    BLOCKSIZE = 65536
    hasher = hashlib.sha256()
    buf = file.read(BLOCKSIZE)
    while len(buf) > 0:
        hasher.update(buf)
        buf = file.read(BLOCKSIZE)
    file.seek(0)
    return hasher.hexdigest()


def get_md5sum(file: IO) -> str:
    """Return the md5sum hash of the file"""
    if type(file) is InMemoryUploadedFile or type(file) is TemporaryUploadedFile:
        return _get_md5sum_memory_file(file)
    else:
        return _get_md5sum_file(file)


def _get_md5sum_memory_file(
    file: Union[InMemoryUploadedFile, TemporaryUploadedFile]
) -> str:
    BLOCKSIZE = 65536
    hasher = hashlib.md5()

    for chunk in file.chunks(BLOCKSIZE):
        hasher.update(chunk)

    file.seek(0)
    return hasher.hexdigest()


def _get_md5sum_file(file: IO) -> str:
    BLOCKSIZE = 65536
    hasher = hashlib.md5()

    buf = file.read(BLOCKSIZE)
    while len(buf) > 0:
        hasher.update(buf)
        buf = file.read(BLOCKSIZE)
    file.seek(0)
    return hasher.hexdigest()


def strip_json_null_bytes(file: IO) -> IO:
    """Return JSON string stream without NULL chars."""
    if type(file) is InMemoryUploadedFile or type(file) is TemporaryUploadedFile:
        return _strip_json_null_bytes_memory_file(file)
    else:
        return _strip_json_null_bytes_file(file)


def _strip_json_null_bytes_memory_file(file: IO) -> IO:
    result = io.BytesIO()
    BLOCKSIZE = 65536

    for chunk in file.chunks(BLOCKSIZE):
        result.write(chunk.decode().replace(r"\u0000", "").encode())
    file.seek(0)
    result.seek(0)

    return result


def _strip_json_null_bytes_file(file: IO) -> IO:
    result = io.BytesIO()
    BLOCKSIZE = 65536

    buf = file.read(BLOCKSIZE)
    while len(buf) > 0:
        result.write(buf.decode().replace(r"\u0000", "").encode())
        buf = file.read(BLOCKSIZE)
    file.seek(0)
    result.seek(0)

    return result


def safe_join(base: str, *paths: str) -> str:
    """
    A version of django.utils._os.safe_join for S3 paths.
    Joins one or more path components to the base path component
    intelligently. Returns a normalized version of the final path.
    The final path must be located inside of the base path component
    (otherwise a ValueError is raised).
    Paths outside the base path indicate a possible security
    sensitive operation.
    """
    base_path = base
    base_path = base_path.rstrip("/")
    paths = tuple(paths)

    final_path = base_path + "/"
    for path in paths:
        _final_path = posixpath.normpath(posixpath.join(final_path, path))
        # posixpath.normpath() strips the trailing /. Add it back.
        if path.endswith("/") or _final_path + "/" == final_path:
            _final_path += "/"
        final_path = _final_path
    if final_path == base_path:
        final_path += "/"

    # Ensure final_path starts with base_path and that the next character after
    # the base path is /.
    base_path_len = len(base_path)
    if not final_path.startswith(base_path) or final_path[base_path_len] != "/":
        raise ValueError(
            "the joined path is located outside of the base path" " component"
        )

    return final_path.lstrip("/")


def is_qgis_project_file(filename: str) -> bool:
    """Returns whether the filename seems to be a QGIS project file by checking the file extension."""
    path = PurePath(filename)

    if path.suffix.lower() in (".qgs", ".qgz"):
        return True

    return False


def get_qgis_project_file(project_id: str) -> Optional[str]:
    """Return the relative path inside the project of the qgs/qgz file or
    None if no qgs/qgz file is present"""

    bucket = get_s3_bucket()

    prefix = "projects/{}/files/".format(project_id)

    for obj in bucket.objects.filter(Prefix=prefix):
        if is_qgis_project_file(obj.key):
            path = PurePath(obj.key)
            return str(path.relative_to(*path.parts[:3]))

    return None


def check_s3_key(key: str) -> Optional[str]:
    """Check to see if an object exists on S3. It it exists, the function
    returns the sha256 of the file from the metadata"""

    client = get_s3_client()
    try:
        head = client.head_object(Bucket=settings.STORAGE_BUCKET_NAME, Key=key)
    except ClientError as e:
        if e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404:
            return None
        else:
            raise e

    metadata = head["Metadata"]
    if "sha256sum" in metadata:
        return metadata["sha256sum"]
    else:
        return metadata["Sha256sum"]


def get_deltafile_schema_validator() -> jsonschema.Draft7Validator:
    """Creates a JSON schema validator to check whether the provided delta
    file is valid.

    Returns:
        jsonschema.Draft7Validator -- JSON Schema validator
    """
    schema_file = os.path.join(
        os.path.dirname(os.path.abspath(__file__)), "deltafile_01.json"
    )

    with open(schema_file) as f:
        schema_dict = json.load(f)

    jsonschema.Draft7Validator.check_schema(schema_dict)

    return jsonschema.Draft7Validator(schema_dict)


def get_s3_project_size(project_id: str) -> int:
    """Return the size in MB of the project on the storage, including the
    exported files and their versions"""

    bucket = get_s3_bucket()

    prefix = f"projects/{project_id}/"

    total_size = 0
    for version in bucket.object_versions.filter(Prefix=prefix):
        total_size += version.size or 0

    return round(total_size / (1000 * 1000), 3)


def get_project_files(project_id: str, path: str = "") -> Iterable[S3Object]:
    """Returns a list of files and their versions.

    Args:
        project_id (str): the project id
        path (str): additional filter prefix

    Returns:
        Iterable[S3ObjectWithVersions]: the list of files
    """
    bucket = get_s3_bucket()
    prefix = f"projects/{project_id}/files/{path}"

    return list_files(bucket, prefix, strip_prefix=True)


def get_project_files_with_versions(project_id: str) -> Iterable[S3ObjectWithVersions]:
    """Returns a list of files and their versions.

    Args:
        project_id (str): the project id

    Returns:
        Iterable[S3ObjectWithVersions]: the list of files
    """
    bucket = get_s3_bucket()
    prefix = f"projects/{project_id}/files/"

    return list_files_with_versions(bucket, prefix, strip_prefix=True)


def get_project_file_with_versions(
    project_id: str, filename: str
) -> Optional[S3ObjectWithVersions]:
    """Returns a list of files and their versions.

    Args:
        project_id (str): the project id

    Returns:
        Iterable[S3ObjectWithVersions]: the list of files
    """
    bucket = get_s3_bucket()
    prefix = f"projects/{project_id}/files/{filename}"
    files = [
        f
        for f in list_files_with_versions(bucket, prefix, strip_prefix=True)
        if f.latest.key == prefix
    ]

    return files[0] if files else None


def get_project_package_files(project_id: str, package_id: str) -> Iterable[S3Object]:
    """Returns a list of package files.

    Args:
        project_id (str): the project id

    Returns:
        Iterable[S3ObjectWithVersions]: the list of package files
    """
    bucket = get_s3_bucket()
    prefix = f"projects/{project_id}/packages/{package_id}/"

    return list_files(bucket, prefix, strip_prefix=True)


def get_project_files_count(project_id: str) -> int:
    """Returns the number of files within a project."""
    bucket = get_s3_bucket()
    prefix = f"projects/{project_id}/files/"
    files = list(bucket.objects.filter(Prefix=prefix))

    return len(files)


def get_project_package_files_count(project_id: str) -> int:
    """Returns the number of package files within a project."""
    bucket = get_s3_bucket()
    prefix = f"projects/{project_id}/export/"
    files = list(bucket.objects.filter(Prefix=prefix))

    return len(files)


def get_s3_object_url(
    key: str, bucket: mypy_boto3_s3.service_resource.Bucket = get_s3_bucket()
) -> str:
    """Returns the block storage URL for a given key. The key may not exist in the bucket.

    Args:
        key (str): the object key
        bucket (boto3.Bucket, optional): Bucket for that object. Defaults to `get_s3_bucket()`.

    Returns:
        str: URL
    """
    return f"{settings.STORAGE_ENDPOINT_URL}/{bucket.name}/{key}"


def list_files(
    bucket: mypy_boto3_s3.service_resource.Bucket,
    prefix: str,
    strip_prefix: bool = True,
) -> List[S3Object]:
    """Iterator that lists a bucket's objects under prefix."""
    files = []
    for f in bucket.objects.filter(Prefix=prefix):
        if strip_prefix:
            start_idx = len(prefix)
            name = f.key[start_idx:]
        else:
            name = f.key

        files.append(
            S3Object(
                name=name,
                key=f.key,
                last_modified=f.last_modified,
                size=f.size,
                etag=f.e_tag,
                md5sum=f.e_tag.replace('"', ""),
            )
        )

    files.sort(key=lambda f: f.name)

    return files


def list_versions(
    bucket: mypy_boto3_s3.service_resource.Bucket,
    prefix: str,
    strip_prefix: bool = True,
) -> List[S3ObjectVersion]:
    """Iterator that lists a bucket's objects under prefix."""
    versions = []
    for v in bucket.object_versions.filter(Prefix=prefix):
        if strip_prefix:
            start_idx = len(prefix)
            name = v.key[start_idx:]
        else:
            name = v.key

        versions.append(S3ObjectVersion(name, v))

    versions.sort(key=lambda v: (v.key, v.last_modified))

    return versions


def list_files_with_versions(
    bucket: mypy_boto3_s3.service_resource.Bucket,
    prefix: str,
    strip_prefix: bool = True,
) -> Iterable[S3ObjectWithVersions]:
    """Yields an object with all it's versions

    Returns:
        Iterable[S3ObjectWithVersions]: an iterator with the objects with their versions

    Yields:
        Iterator[Iterable[S3ObjectWithVersions]]: the object with its versions
    """
    last_key = None
    versions: List[S3ObjectVersion] = []
    latest: Optional[S3ObjectVersion] = None

    for v in list_versions(bucket, prefix, strip_prefix):
        if last_key != v.key:
            if last_key:
                assert latest

                yield S3ObjectWithVersions(latest, versions)

            latest = None
            versions = []
            last_key = v.key

        versions.append(v)

        if v.is_latest:
            latest = v

    if last_key:
        assert latest
        yield S3ObjectWithVersions(latest, versions)


# docker-app/qfieldcloud/core/validators.py

from django.core.exceptions import ValidationError
from django.utils.translation import gettext as _


def reserved_words_validator(value):
    reserved_words = [
        "user",
        "users",
        "project",
        "projects",
        "owner",
        "push",
        "file",
        "files",
        "collaborator",
        "collaborators",
        "member",
        "members",
        "organization",
        "qfield",
        "qfieldcloud",
        "history",
        "version",
        "delta",
        "deltas",
        "deltafile",
        "auth",
        "qfield-files",
        "esri",
    ]
    if value.lower() in reserved_words:
        raise ValidationError(_('"{}" is a reserved word!').format(value))


# docker-app/qfieldcloud/core/tests/test_api.py

import logging
import time

from django.core.cache import cache
from rest_framework import status
from rest_framework.test import APITransactionTestCase

logging.disable(logging.CRITICAL)


class QfcTestCase(APITransactionTestCase):
    def setUp(self):
        # Empty cache value
        cache.delete("status_results")

    def test_api_status(self):
        response = self.client.get("/api/v1/status/")
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(response.json()["redis"], "ok")
        self.assertEqual(response.json()["storage"], "ok")
        self.assertEqual(response.json()["geodb"], "ok")

    def test_api_status_cache(self):
        tic = time.perf_counter()
        self.client.get("/api/v1/status/")
        toc = time.perf_counter()

        self.assertGreater(toc - tic, 0)


# docker-app/qfieldcloud/core/tests/test_delta.py

import io
import json
import logging
import time

import fiona
import rest_framework
from django.http.response import FileResponse, HttpResponse
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core import utils
from qfieldcloud.core.models import (
    Job,
    Organization,
    OrganizationMember,
    Project,
    ProjectCollaborator,
    User,
)
from rest_framework import status
from rest_framework.test import APITransactionTestCase

from .utils import get_filename, setup_subscription_plans, testdata_path

logging.disable(logging.CRITICAL)


class QfcTestCase(APITransactionTestCase):
    def setUp(self):
        setup_subscription_plans()

        # Create a user
        self.user1 = User.objects.create_user(username="user1", password="abc123")
        self.user1.save()
        self.user2 = User.objects.create_user(username="user2", password="abc123")
        self.user2.save()
        self.user3 = User.objects.create_user(username="user3", password="abc123")
        self.user3.save()

        self.token1 = AuthToken.objects.get_or_create(user=self.user1)[0]
        self.token2 = AuthToken.objects.get_or_create(user=self.user2)[0]
        self.token3 = AuthToken.objects.get_or_create(user=self.user3)[0]

        self.org1 = Organization.objects.create(
            username="org1", organization_owner=self.user1
        )

        # Create a project
        self.project1 = Project.objects.create(
            name="project1",
            is_public=False,
            owner=self.org1,
        )
        self.project1.save()

        self.project2 = Project.objects.create(
            name="project2",
            is_public=False,
            owner=self.user2,
        )
        self.project1.save()

        OrganizationMember.objects.create(
            organization=self.org1,
            member=self.user2,
        )
        ProjectCollaborator.objects.create(
            project=self.project1,
            collaborator=self.user2,
            role=ProjectCollaborator.Roles.REPORTER,
        )
        OrganizationMember.objects.create(
            organization=self.org1,
            member=self.user3,
        )
        ProjectCollaborator.objects.create(
            project=self.project1,
            collaborator=self.user3,
            role=ProjectCollaborator.Roles.ADMIN,
        )

    def tearDown(self):
        while True:
            # make sure there are no active jobs in the queue
            if (
                Job.objects.all()
                .filter(
                    status__in=[
                        Job.Status.PENDING,
                        Job.Status.QUEUED,
                        Job.Status.STARTED,
                    ]
                )
                .count()
                == 0
            ):
                time.sleep(1)
                return

    def fail(self, msg: str, job: Job = None):
        if job:
            msg += f"\n\nOutput:\n================\n{job.output}\n================"

            if job.feedback:
                if "error_stack" in job.feedback:
                    msg += "\n\nError:\n================"
                    for single_error_stack in job.feedback["error_stack"]:
                        msg += "\n"
                        msg += single_error_stack

                    msg += f"  {job.feedback['error']}\n================"

                feedback = json.dumps(job.feedback, indent=2, sort_keys=True)
                msg += f"\n\nFeedback:\n================\n{feedback}\n================"
            else:
                msg += "\n\nFeedback: None"

        super().fail(msg)

    def assertHttpOk(self, response: HttpResponse):
        try:
            self.assertTrue(
                rest_framework.status.is_success(response.status_code), response.json()
            )
        except Exception:
            self.assertTrue(
                rest_framework.status.is_success(response.status_code), response.content
            )

    def upload_project_files(self, project) -> Project:
        # Verify the original geojson file
        with open(testdata_path("delta/points.geojson")) as f:
            points_geojson = json.load(f)
            features = sorted(points_geojson["features"], key=lambda k: k["id"])
            self.assertEqual(1, features[0]["properties"]["int"])

        for project_file in [
            "points.geojson",
            "polygons.geojson",
            "testdata.gpkg",
            "project.qgs",
            "nonspatial.csv",
        ]:
            file_path = testdata_path(f"delta/{project_file}")
            response = self.client.post(
                f"/api/v1/files/{project.id}/{project_file}/",
                {"file": open(file_path, "rb")},
                format="multipart",
            )
            self.assertTrue(
                status.is_success(response.status_code),
                f"Failed to upload file '{project_file}'",
            )

        # wait until the project file check are ready
        for i in range(30):
            updated_project = Project.objects.get(id=project.id)
            if updated_project.project_filename:
                return updated_project

            time.sleep(1)

        raise Exception("Projectfile never set on project")

    def test_push_apply_delta_file(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        self.upload_and_check_deltas(
            project=project,
            delta_filename="singlelayer_singledelta2.json",
            token=self.token1.key,
            final_values=[
                [
                    "c8c421cd-e39c-40a0-97d8-a319c245ba14",
                    "STATUS_APPLIED",
                    self.user1.username,
                ]
            ],
        )

        gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg"))
        with fiona.open(gpkg, layer="points") as layer:
            features = list(layer)
            self.assertEqual(666, features[0]["properties"]["int"])

    def test_push_apply_delta_file_with_null_char(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        self.upload_and_check_deltas(
            project=project,
            delta_filename="singlelayer_singledelta_null.json",
            token=self.token1.key,
            final_values=[
                [
                    "9311eb96-bff8-4d5b-ab36-c314a007cfcd",
                    "STATUS_APPLIED",
                    self.user1.username,
                ]
            ],
        )

        gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg"))
        with fiona.open(gpkg, layer="points") as layer:
            features = list(layer)
            self.assertEqual("", features[0]["properties"]["str"])

    def test_push_apply_delta_file_with_error(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        self.upload_and_check_deltas(
            project=project,
            delta_filename="with_errors.json",
            token=self.token1.key,
            final_values=[
                [
                    "65b605b4-9832-4de0-9055-92e1dd94ebec",
                    "STATUS_NOT_APPLIED",
                    self.user1.username,
                ]
            ],
        )

    def test_push_apply_delta_file_invalid_json_schema(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        bucket = utils.get_s3_bucket()
        prefix = utils.safe_join(f"projects/{project.id}/deltas/")
        wrong_deltas_before = list(bucket.objects.filter(Prefix=prefix))
        delta_file = testdata_path("delta/deltas/not_schema_valid.json")

        self.assertFalse(self.upload_deltas(project, "not_schema_valid.json"))

        # check it is uploaded
        wrong_deltas = list(bucket.objects.filter(Prefix=prefix))

        # TODO : cleanup buckets before in setUp so tests are completely independent
        self.assertEqual(len(wrong_deltas), len(wrong_deltas_before) + 1)

        f = self.get_delta_file_with_project_id(self.project1, delta_file)
        self.assertEqual(wrong_deltas[-1].get()["Body"].read().decode(), f.read())

    def test_push_apply_delta_file_not_json(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        delta_file = testdata_path("file.txt")

        response = self.client.post(
            f"/api/v1/deltas/{project.id}/",
            {"file": open(delta_file, "r")},
            format="multipart",
        )

        self.assertFalse(rest_framework.status.is_success(response.status_code))

    def test_push_apply_delta_file_conflicts_overwrite_true(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        self.upload_and_check_deltas(
            project=project,
            delta_filename="singlelayer_singledelta_conflict.json",
            token=self.token1.key,
            final_values=[
                [
                    "8d185b67-f05e-40c6-9c9a-6ceca8100c39",
                    "STATUS_APPLIED",
                    self.user1.username,
                ]
            ],
        )

    def test_push_apply_delta_file_twice(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        self.upload_and_check_deltas(
            project=project,
            delta_filename="singlelayer_singledelta.json",
            token=self.token1.key,
            final_values=[
                [
                    "9311eb96-bff8-4d5b-ab36-c314a007cfcd",
                    "STATUS_APPLIED",
                    self.user1.username,
                ]
            ],
        )

        gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg"))
        with fiona.open(gpkg, layer="points") as layer:
            features = list(layer)
            self.assertEqual(666, features[0]["properties"]["int"])

        self.upload_and_check_deltas(
            project=project,
            delta_filename="singlelayer_singledelta.json",
            token=self.token1.key,
            final_values=[
                [
                    "9311eb96-bff8-4d5b-ab36-c314a007cfcd",
                    "STATUS_APPLIED",
                    self.user1.username,
                ]
            ],
        )

        self.assertTrue(
            self.upload_deltas(project, "singlelayer_singledelta_diff_content.json")
        )

    def test_push_list_deltas(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        self.assertTrue(
            self.upload_deltas(self.project1, "singlelayer_singledelta3.json")
        )
        self.assertTrue(
            self.upload_deltas(self.project1, "singlelayer_singledelta4.json")
        )

        self.upload_and_check_deltas(
            project=project,
            delta_filename=None,
            token=self.token1.key,
            final_values=[
                [
                    "802ae2ef-f360-440e-a816-8990d6a06667",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
                [
                    "e4546ec2-6e01-43a1-ab30-a52db9469afd",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
            ],
        )

    def test_push_list_multidelta(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        self.upload_and_check_deltas(
            project=project,
            delta_filename="singlelayer_multidelta.json",
            token=self.token1.key,
            final_values=[
                [
                    "736bf2c2-646a-41a2-8c55-28c26aecd68d",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
                [
                    "8adac0df-e1d3-473e-b150-f8c4a91b4781",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
                [
                    "c6c88e78-172c-4f77-b2fd-2ff41f5aa854",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
            ],
        )

    def test_list_all_deltas_and_list_deltas_by_deltafile(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        self.assertTrue(
            self.upload_deltas(self.project1, "singlelayer_singledelta5.json")
        )
        self.assertTrue(
            self.upload_deltas(self.project1, "singlelayer_singledelta6.json")
        )

        # check all the deltas
        self.upload_and_check_deltas(
            project=project,
            delta_filename=None,
            token=self.token1.key,
            final_values=[
                [
                    "ad98634e-509f-4dff-9000-de79b09c5359",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
                [
                    "df6a19eb-7d61-4c64-9e3b-29bce0a8dfab",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
            ],
        )

        # check all the deltas from a single file
        self.upload_and_check_deltas(
            project=project,
            delta_filename=None,
            token=self.token1.key,
            final_values=[
                [
                    "ad98634e-509f-4dff-9000-de79b09c5359",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
            ],
            deltafile_id="3aab7e58-ea27-4b7c-9bca-c772b6d94820",
        )

    def test_push_apply_delta_file_conflicts_overwrite_false(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        # Set the overwrite_conflicts flag to False
        project.overwrite_conflicts = False
        project.save()

        self.upload_and_check_deltas(
            project=project,
            delta_filename="singlelayer_singledelta_conflict2.json",
            token=self.token1.key,
            final_values=[
                [
                    "bd507a3d-aa7b-42c4-bdb7-23ff34f65d5c",
                    "STATUS_CONFLICT",
                    self.user1.username,
                ]
            ],
        )

    def test_list_deltas_unexisting_project(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        self.upload_project_files(self.project1)

        response = self.client.get(
            "/api/v1/deltas/7199612e-7641-48fc-8c11-c25176a9761b/"
        )
        self.assertFalse(status.is_success(response.status_code))
        json = response.json()
        self.assertEqual(json["code"], "object_not_found")

    def test_push_delta_not_allowed(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token2.key)
        project = self.upload_project_files(self.project1)

        self.upload_and_check_deltas(
            project=project,
            delta_filename="singlelayer_multidelta_patch_create.json",
            token=self.token2.key,
            final_values=[
                [
                    "736bf2c2-646a-41a2-8c55-28c26aecd68d",
                    "STATUS_UNPERMITTED",
                    self.user2.username,
                ],
                [
                    "8adac0df-e1d3-473e-b150-f8c4a91b4781",
                    "STATUS_APPLIED",
                    self.user2.username,
                ],
            ],
        )

    def test_non_spatial_delta(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        # Push a deltafile
        self.upload_and_check_deltas(
            project=project,
            delta_filename="nonspatial.json",
            token=self.token1.key,
            final_values=[
                [
                    "1270b97d-6a28-49cc-83f3-b827ec574fee",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
                [
                    "f326c3c1-138f-4261-9151-4946237ce714",
                    "STATUS_APPLIED",
                    self.user1.username,
                ],
            ],
        )

        self.assertEqual(
            self.get_file_contents(project, "nonspatial.csv"), b'fid,col1\n"1",qux\n'
        )

    def test_delta_pushed_after_job_triggered(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        # Push a deltafile
        deltafile1_name = "singlelayer_singledelta.json"
        self.assertTrue(self.upload_deltas(project, deltafile1_name))
        with open(testdata_path(f"delta/deltas/{deltafile1_name}")) as f:
            deltafile1_id = json.load(f)["id"]

        deltafile2_name = "singlelayer_singledelta2.json"
        self.assertTrue(self.upload_deltas(project, deltafile2_name))
        with open(testdata_path(f"delta/deltas/{deltafile2_name}")) as f:
            deltafile2_id = json.load(f)["id"]

        self.check_deltas_by_file_id(
            project,
            deltafile1_id,
            final_values=[
                [
                    "9311eb96-bff8-4d5b-ab36-c314a007cfcd",
                    "STATUS_APPLIED",
                    self.user1.username,
                ]
            ],
            token=self.token1.key,
        )

        self.check_deltas_by_file_id(
            project,
            deltafile2_id,
            final_values=[
                [
                    "c8c421cd-e39c-40a0-97d8-a319c245ba14",
                    "STATUS_APPLIED",
                    self.user1.username,
                ]
            ],
            token=self.token1.key,
        )

    def test_delta_pushed_after_job_triggered_two_projects(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project1 = self.upload_project_files(self.project1)

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token2.key)
        project2 = self.upload_project_files(self.project2)

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Push a deltafile
        deltafile1_name = "singlelayer_singledelta.json"
        self.assertTrue(self.upload_deltas(project1, deltafile1_name))
        with open(testdata_path(f"delta/deltas/{deltafile1_name}")) as f:
            deltafile1_id = json.load(f)["id"]

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token2.key)
        deltafile2_name = "singlelayer_singledelta_project2.json"
        self.assertTrue(self.upload_deltas(project2, deltafile2_name))
        with open(testdata_path(f"delta/deltas/{deltafile2_name}")) as f:
            deltafile2_id = json.load(f)["id"]

        self.check_deltas_by_file_id(
            project1,
            deltafile1_id,
            final_values=[
                [
                    "9311eb96-bff8-4d5b-ab36-c314a007cfcd",
                    "STATUS_APPLIED",
                    self.user1.username,
                ]
            ],
            token=self.token1.key,
        )

        self.check_deltas_by_file_id(
            project2,
            deltafile2_id,
            final_values=[
                [
                    "f2af4942-e4ab-446e-bd97-5aab17e7ccc1",
                    "STATUS_APPLIED",
                    self.user2.username,
                ]
            ],
            token=self.token2.key,
        )

    def test_change_and_delete_pushed_only_features(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        project = self.upload_project_files(self.project1)

        # 1) client 1 creates a feature
        self.upload_and_check_deltas(
            project=project,
            delta_filename="multistage_p1_c1_create.json",
            token=self.token1.key,
            final_values=[
                [
                    "9311eb96-bff8-4d5b-ab36-c314a007cfcd",
                    "STATUS_APPLIED",
                    self.user1.username,
                ]
            ],
        )

        gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg"))
        with fiona.open(gpkg, "r", layer="points") as layer:
            features = list(layer)

            self.assertEqual(len(features), 4)
            self.assertEqual(features[0]["properties"]["int"], 1)
            self.assertEqual(features[1]["properties"]["int"], 2)
            self.assertEqual(features[2]["properties"]["int"], 3)
            self.assertEqual(features[3]["properties"]["int"], 1000)

        # 2) client 2 creates a feature
        self.upload_and_check_deltas(
            project=project,
            delta_filename="multistage_p2_c2_create.json",
            token=self.token3.key,
            final_values=[
                [
                    "608bbfb7-fb9c-49c4-818f-f636ee4ec20a",
                    "STATUS_APPLIED",
                    self.user3.username,
                ]
            ],
        )

        gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg"))
        with fiona.open(gpkg, "r", layer="points") as layer:
            features = list(layer)

            self.assertEqual(len(features), 5)
            self.assertEqual(features[0]["properties"]["int"], 1)
            self.assertEqual(features[1]["properties"]["int"], 2)
            self.assertEqual(features[2]["properties"]["int"], 3)
            self.assertEqual(features[3]["properties"]["int"], 1000)
            self.assertEqual(features[4]["properties"]["int"], 2000)

        # 3) client 1 updates their created feature
        self.upload_and_check_deltas(
            project=project,
            delta_filename="multistage_p3_c1_patch.json",
            token=self.token3.key,
            final_values=[
                [
                    "f11603e5-13b2-43a9-b27a-db722297773b",
                    "STATUS_APPLIED",
                    self.user3.username,
                ]
            ],
        )

        gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg"))
        with fiona.open(gpkg, "r", layer="points") as layer:
            features = list(layer)

            self.assertEqual(len(features), 5)
            self.assertEqual(features[0]["properties"]["int"], 1)
            self.assertEqual(features[1]["properties"]["int"], 2)
            self.assertEqual(features[2]["properties"]["int"], 3)
            self.assertEqual(features[3]["properties"]["int"], 1001)
            self.assertEqual(features[4]["properties"]["int"], 2000)

        # 4) client 2 updates their created feature
        self.upload_and_check_deltas(
            project=project,
            delta_filename="multistage_p4_c2_patch.json",
            token=self.token3.key,
            final_values=[
                [
                    "582de6de-562f-4482-9350-5b5aaa25d822",
                    "STATUS_APPLIED",
                    self.user3.username,
                ]
            ],
        )

        gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg"))
        with fiona.open(gpkg, "r", layer="points") as layer:
            features = list(layer)

            self.assertEqual(len(features), 5)
            self.assertEqual(features[0]["properties"]["int"], 1)
            self.assertEqual(features[1]["properties"]["int"], 2)
            self.assertEqual(features[2]["properties"]["int"], 3)
            self.assertEqual(features[3]["properties"]["int"], 1001)
            self.assertEqual(features[4]["properties"]["int"], 2002)

        # 5) client 1 deletes their created feature
        self.upload_and_check_deltas(
            project=project,
            delta_filename="multistage_p5_c1_delete.json",
            token=self.token3.key,
            final_values=[
                [
                    "b7a09a1d-9626-4da0-8456-61c2ff884611",
                    "STATUS_APPLIED",
                    self.user3.username,
                ]
            ],
        )

        gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg"))
        with fiona.open(gpkg, "r", layer="points") as layer:
            features = list(layer)

            self.assertEqual(len(features), 4)
            self.assertEqual(features[0]["properties"]["int"], 1)
            self.assertEqual(features[1]["properties"]["int"], 2)
            self.assertEqual(features[2]["properties"]["int"], 3)
            self.assertEqual(features[3]["properties"]["int"], 2002)

        # 6) client 2 deletes their created feature
        self.upload_and_check_deltas(
            project=project,
            delta_filename="multistage_p6_c2_delete.json",
            token=self.token3.key,
            final_values=[
                [
                    "7cb988e9-5de2-4bd7-af4b-c1d27a2d579f",
                    "STATUS_APPLIED",
                    self.user3.username,
                ]
            ],
        )

        gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg"))
        with fiona.open(gpkg, "r", layer="points") as layer:
            features = list(layer)

            self.assertEqual(len(features), 3)
            self.assertEqual(features[0]["properties"]["int"], 1)
            self.assertEqual(features[1]["properties"]["int"], 2)
            self.assertEqual(features[2]["properties"]["int"], 3)

    def get_file_contents(self, project, filename):
        response = self.client.get(f"/api/v1/files/{project.id}/{filename}/")

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(get_filename(response), filename)

        if isinstance(response, FileResponse):
            return b"".join(response.streaming_content)
        else:
            return response.content

    def get_delta_file_with_project_id(self, project, delta_filename):
        """Retrieves a delta json file with the project id replaced by the project.id"""
        with open(delta_filename, "r") as f:
            deltafile = json.load(f)
            deltafile["project"] = str(project.id)
            json_str = json.dumps(deltafile)
            return io.StringIO(json_str)

    def upload_deltas(self, project, delta_filename):
        delta_file = testdata_path(f"delta/deltas/{delta_filename}")

        response = self.client.post(
            f"/api/v1/deltas/{project.id}/",
            {"file": self.get_delta_file_with_project_id(project, delta_file)},
            format="multipart",
        )
        return rest_framework.status.is_success(response.status_code)

    def upload_and_check_deltas(
        self,
        project,
        delta_filename,
        final_values,
        token,
        wait_status=["STATUS_PENDING", "STATUS_BUSY"],
        failing_status=["STATUS_ERROR"],
        immediate_values=None,
        deltafile_id=None,
    ):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + token)

        if delta_filename is not None:
            # Push a deltafile
            self.assertTrue(self.upload_deltas(project, delta_filename))

            delta_file = testdata_path(f"delta/deltas/{delta_filename}")

            if not deltafile_id:
                with open(delta_file) as f:
                    deltafile_id = json.load(f)["id"]

        self.check_deltas_by_file_id(
            project,
            deltafile_id,
            final_values,
            token,
            wait_status,
            failing_status,
            immediate_values,
        )

    def check_deltas_by_file_id(
        self,
        project,
        deltafile_id,
        final_values,
        token,
        wait_status=["STATUS_PENDING", "STATUS_BUSY"],
        failing_status=["STATUS_ERROR"],
        immediate_values=None,
    ):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + token)

        uri = f"/api/v1/deltas/{project.id}/"
        if deltafile_id:
            uri = f"{uri}{deltafile_id}/"

        response = self.client.get(uri)
        self.assertTrue(rest_framework.status.is_success(response.status_code))
        payload = response.json()
        payload = sorted(payload, key=lambda k: k["id"])

        if immediate_values:
            self.assertEqual(len(payload), len(immediate_values))

            for idx, immediate_value in enumerate(immediate_values):
                delta_id, status, created_by = immediate_value
                status = status if isinstance(status, list) else list(status)

                self.assertEqual(payload[idx]["id"], delta_id)
                self.assertIn(payload[idx]["status"], status)
                self.assertEqual(payload[idx]["created_by"], created_by)

        job = Job.objects.filter(
            project=self.project1,
            type=Job.Type.DELTA_APPLY,
        ).latest("updated_at")

        for _ in range(10):

            time.sleep(2)
            response = self.client.get(uri)

            self.assertHttpOk(response)

            payload = response.json()
            payload = sorted(payload, key=lambda k: k["id"])

            self.assertEqual(len(payload), len(final_values))

            for idx, final_value in enumerate(final_values):
                if payload[idx]["status"] in wait_status:
                    break

                if payload[idx]["status"] in failing_status:
                    job.refresh_from_db()
                    self.fail(f"Got failing status {payload[idx]['status']}", job=job)
                    return

                delta_id, status, created_by = final_value
                status = status if isinstance(status, list) else [status]

                self.assertEqual(payload[idx]["id"], delta_id)
                self.assertIn(payload[idx]["status"], status)
                self.assertEqual(payload[idx]["created_by"], created_by)

                if len(final_values) == idx + 1:
                    return

        self.fail("Worker didn't finish", job=job)


# docker-app/qfieldcloud/core/tests/test_geodb.py

import logging
import unittest

import psycopg2
from qfieldcloud.core.models import Geodb, User

from .utils import setup_subscription_plans

logging.disable(logging.CRITICAL)


class QfcTestCase(unittest.TestCase):
    def setUp(self):
        setup_subscription_plans()

    def test_create_db(self):
        # Create a user
        user1 = User.objects.create_user(
            username="user1", password="abc123", email="user1@pizza.it"
        )

        # Create a geodb object

        geodb = Geodb.objects.create(
            user=user1,
            hostname="geodb",
            port=5432,
        )

        conn = psycopg2.connect(
            dbname=geodb.dbname,
            user=geodb.username,
            password=geodb.password,
            host=geodb.hostname,
            port=geodb.port,
        )

        cur = conn.cursor()

        cur.execute(
            """
            CREATE TABLE pizza (
                code        char(5) CONSTRAINT firstkey PRIMARY KEY,
                title       varchar(40) NOT NULL
            );
            """
        )

        conn.commit()


# docker-app/qfieldcloud/core/tests/test_organization.py

import calendar
import logging
import uuid
from datetime import timedelta

from django.utils.timezone import now
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.models import (
    Delta,
    Job,
    Organization,
    OrganizationMember,
    Project,
    User,
)
from rest_framework import status
from rest_framework.test import APITestCase

from .utils import setup_subscription_plans

logging.disable(logging.CRITICAL)


class QfcTestCase(APITestCase):
    def setUp(self):
        setup_subscription_plans()

        # Create a user
        self.user1 = User.objects.create_user(username="user1", password="abc123")
        self.token1 = AuthToken.objects.get_or_create(user=self.user1)[0]

        # Create a user
        self.user2 = User.objects.create_user(username="user2", password="abc123")
        self.token2 = AuthToken.objects.get_or_create(user=self.user2)[0]

        # Create a user
        self.user3 = User.objects.create_user(username="user3", password="abc123")
        self.token3 = AuthToken.objects.get_or_create(user=self.user3)[0]

        # Create an organization
        self.organization1 = Organization.objects.create(
            username="organization1",
            password="abc123",
            user_type=2,
            organization_owner=self.user1,
        )

    def test_list_members(self):

        # Set user2 as member of organization1
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user2,
            role=OrganizationMember.Roles.MEMBER,
        )

        # Set user3 as admin of organization1
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user3,
            role=OrganizationMember.Roles.ADMIN,
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.get("/api/v1/members/organization1/")

        self.assertTrue(status.is_success(response.status_code))

        json = response.json()
        json = sorted(json, key=lambda k: k["member"])

        self.assertEqual(len(json), 2)
        self.assertEqual(json[0]["member"], "user2")
        self.assertEqual(json[0]["role"], "member")
        self.assertEqual(json[1]["member"], "user3")
        self.assertEqual(json[1]["role"], "admin")

    def test_create_member(self):

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.post(
            "/api/v1/members/organization1/",
            {
                "member": "user2",
                "role": "admin",
            },
        )

        self.assertTrue(status.is_success(response.status_code))

        members = OrganizationMember.objects.all()
        self.assertEqual(len(members), 1)
        self.assertEqual(members[0].organization, self.organization1)
        self.assertEqual(members[0].member, self.user2)
        self.assertEqual(members[0].role, OrganizationMember.Roles.ADMIN)

    def test_update_member(self):
        # Set user2 as member of organization1
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user2,
            role=OrganizationMember.Roles.MEMBER,
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.patch(
            "/api/v1/members/organization1/user2/",
            {
                "role": "member",
            },
        )

        self.assertTrue(status.is_success(response.status_code))

        members = OrganizationMember.objects.all()
        self.assertEqual(len(members), 1)
        self.assertEqual(members[0].organization, self.organization1)
        self.assertEqual(members[0].member, self.user2)
        self.assertEqual(members[0].role, OrganizationMember.Roles.MEMBER)

    def test_delete_member(self):

        # Set user2 as member of organization1
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user2,
            role=OrganizationMember.Roles.MEMBER,
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.delete("/api/v1/members/organization1/user2/")

        self.assertTrue(status.is_success(response.status_code))

        members = OrganizationMember.objects.all()
        self.assertEqual(len(members), 0)

    def test_admin_can_add_member(self):
        # Set user2 as member of organization1
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user2,
            role=OrganizationMember.Roles.MEMBER,
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token2.key)
        response = self.client.post(
            "/api/v1/members/organization1/",
            {
                "member": "user3",
                "role": "member",
            },
        )

        self.assertFalse(status.is_success(response.status_code))

        # Set user2 as admin of organization1
        obj = OrganizationMember.objects.all()[0]
        obj.role = OrganizationMember.Roles.ADMIN
        obj.save()

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token2.key)
        response = self.client.post(
            "/api/v1/members/organization1/",
            {
                "member": "user3",
                "role": "member",
            },
        )

        self.assertTrue(status.is_success(response.status_code))

    def test_billable_users_count(self):
        """Tests billable users calculations"""

        # Set user1 and user2 as member of organization1
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user2,
            role=OrganizationMember.Roles.MEMBER,
        )
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user3,
            role=OrganizationMember.Roles.MEMBER,
        )

        # Create a project owned by the organization
        project1 = Project.objects.create(name="p1", owner=self.organization1)

        def _billable_users_count(base_date=None):
            """Helper to get count of billable users"""
            if base_date is None:
                base_date = now()

            # Note: we can't easily mock dates as the worker can update jobs in the background
            # with the current date
            start_date = base_date.replace(
                day=1, hour=0, minute=0, second=0, microsecond=0
            )
            end_date = base_date.replace(
                day=calendar.monthrange(base_date.year, base_date.month)[1]
            )
            return self.organization1.billable_users(start_date, end_date).count()

        # Initially, there is no billable user
        self.assertEqual(_billable_users_count(), 0)

        # User 1 creates a job
        Job.objects.create(
            project=project1,
            created_by=self.user2,
        )
        # There is now 1 billable user
        self.assertEqual(_billable_users_count(), 1)

        # User 1 creates a delta
        Delta.objects.create(
            deltafile_id=uuid.uuid4(),
            project=project1,
            content="delta",
            created_by=self.user2,
        )
        # There is still 1 billable user
        self.assertEqual(_billable_users_count(), 1)

        # User 2 creates a job
        Job.objects.create(
            project=project1,
            created_by=self.user3,
        )
        # There is 2 billable user
        self.assertEqual(_billable_users_count(), 2)

        # User 2 leaves the organization
        OrganizationMember.objects.filter(member=self.user3).delete()

        # There is now 1 billable user
        self.assertEqual(_billable_users_count(), 1)

        # Report at a different time is empty
        self.assertEqual(_billable_users_count(now() + timedelta(days=365)), 0)


# docker-app/qfieldcloud/core/tests/test_packages.py

import json
import logging
import os
import tempfile
import time
from typing import List, Tuple

import psycopg2
from django.http import FileResponse
from django.utils import timezone
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.geodb_utils import delete_db_and_role
from qfieldcloud.core.models import Geodb, Job, Project, Secret, User
from rest_framework import status
from rest_framework.test import APITransactionTestCase

from .utils import setup_subscription_plans, testdata_path

logging.disable(logging.CRITICAL)


class QfcTestCase(APITransactionTestCase):
    def setUp(self):
        setup_subscription_plans()

        # Create a user
        self.user1 = User.objects.create_user(username="user1", password="abc123")

        self.user2 = User.objects.create_user(username="user2", password="abc123")

        self.token1 = AuthToken.objects.get_or_create(user=self.user1)[0]

        # Create a project
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        try:
            delete_db_and_role("test", self.user1.username)
        except Exception:
            pass

        self.geodb = Geodb.objects.create(
            user=self.user1,
            dbname="test",
            hostname="geodb",
            port=5432,
        )

        self.conn = psycopg2.connect(
            dbname="test",
            user=os.environ.get("GEODB_USER"),
            password=os.environ.get("GEODB_PASSWORD"),
            host="geodb",
            port=5432,
        )

    def tearDown(self):
        self.conn.close()

    def upload_files(
        self,
        token: str,
        project: Project,
        files: List[Tuple[str, str]],
    ):
        self.client.credentials(HTTP_AUTHORIZATION=f"Token {token}")
        for local_filename, remote_filename in files:
            if not local_filename:
                continue

            file = testdata_path(local_filename)
            response = self.client.post(
                f"/api/v1/files/{project.id}/{remote_filename}/",
                {"file": open(file, "rb")},
                format="multipart",
            )
            self.assertTrue(status.is_success(response.status_code))

    def wait_for_project_ok_status(self, project: Project, wait_s: int = 30):
        jobs = Job.objects.filter(project=project).exclude(
            status__in=[Job.Status.FAILED, Job.Status.FINISHED]
        )

        if jobs.count() == 0:
            return

        has_no_pending_jobs = False
        for _ in range(wait_s):
            if (
                Job.objects.filter(project=project, status=Job.Status.PENDING).count()
                == 0
            ):
                has_no_pending_jobs = True

            time.sleep(1)

        if not has_no_pending_jobs:
            self.fail(f"Still pending jobs after waiting for {wait_s} seconds")

        for _ in range(wait_s):
            project.refresh_from_db()
            if project.status == Project.Status.OK:
                return
            if project.status == Project.Status.FAILED:
                self.fail("Waited for ok status, but got failed")
                return

            time.sleep(1)

        self.fail(f"Waited for ok status for {wait_s} seconds")

    def upload_files_and_check_package(
        self,
        token: str,
        project: Project,
        files: List[Tuple[str, str]],
        expected_files: List[str],
        job_create_error: Tuple[int, str] = None,
        tempdir: str = None,
        invalid_layers: List[str] = [],
    ):
        self.client.credentials(HTTP_AUTHORIZATION=f"Token {token}")

        self.upload_files(token, project, files)

        before_started_ts = timezone.now()

        response = self.client.post(
            "/api/v1/jobs/",
            {
                "project_id": project.id,
                "type": Job.Type.PACKAGE,
            },
        )

        if job_create_error:
            self.assertEqual(response.status_code, job_create_error[0])
            self.assertEqual(response.json()["code"], job_create_error[1])
            return
        else:
            self.assertTrue(status.is_success(response.status_code))

        job_id = response.json().get("id")

        # Wait for the worker to finish
        for _ in range(20):
            time.sleep(3)
            response = self.client.get(f"/api/v1/jobs/{job_id}/")
            payload = response.json()

            if payload["status"] == Job.Status.FINISHED:
                response = self.client.get(f"/api/v1/packages/{project.id}/latest/")
                package_payload = response.json()

                self.assertLess(
                    package_payload["packaged_at"], timezone.now().isoformat()
                )
                self.assertGreater(
                    package_payload["packaged_at"],
                    before_started_ts.isoformat(),
                )

                sorted_downloaded_files = [
                    f["name"]
                    for f in sorted(package_payload["files"], key=lambda k: k["name"])
                ]
                sorted_expected_files = sorted(expected_files)

                self.assertListEqual(sorted_downloaded_files, sorted_expected_files)

                if tempdir:
                    for filename in expected_files:
                        response = self.client.get(
                            f"/api/v1/packages/{self.project1.id}/latest/files/project_qfield.qgs/"
                        )
                        local_file = os.path.join(tempdir, filename)

                        self.assertIsInstance(response, FileResponse)

                        with open(local_file, "wb") as f:
                            for chunk in response.streaming_content:
                                f.write(chunk)

                for layer_id in package_payload["layers"]:
                    layer_data = package_payload["layers"][layer_id]

                    if layer_id in invalid_layers:
                        self.assertFalse(layer_data["is_valid"], layer_id)
                    else:
                        self.assertTrue(layer_data["is_valid"], layer_id)

                return
            elif payload["status"] == Job.Status.FAILED:
                print(
                    "Job feedback:",
                    json.dumps(
                        Job.objects.get(id=job_id).feedback, sort_keys=True, indent=2
                    ),
                )
                self.fail("Worker failed with error")

        self.fail("Worker didn't finish")

    def test_list_files_for_qfield(self):
        cur = self.conn.cursor()
        cur.execute("CREATE TABLE point (id integer, geometry geometry(point, 2056))")
        self.conn.commit()
        cur.execute(
            "INSERT INTO point(id, geometry) VALUES(1, ST_GeomFromText('POINT(2725505 1121435)', 2056))"
        )
        self.conn.commit()

        self.upload_files_and_check_package(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/project2.qgs", "project.qgs"),
                ("delta/points.geojson", "points.geojson"),
            ],
            expected_files=["data.gpkg", "project_qfield.qgs"],
        )

    def test_list_files_missing_qgis_project_file(self):
        self.upload_files_and_check_package(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/points.geojson", "points.geojson"),
            ],
            job_create_error=(400, "no_qgis_project"),
            expected_files=[],
        )

    def test_project_never_packaged(self):
        self.upload_files(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/project2.qgs", "project.qgs"),
            ],
        )

        response = self.client.get(f"/api/v1/packages/{self.project1.id}/latest/")

        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.json()["code"], "invalid_job")

    def test_download_file_for_qfield(self):
        tempdir = tempfile.mkdtemp()

        self.upload_files_and_check_package(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/nonspatial.csv", "nonspatial.csv"),
                ("delta/testdata.gpkg", "testdata.gpkg"),
                ("delta/points.geojson", "points.geojson"),
                ("delta/polygons.geojson", "polygons.geojson"),
                ("delta/project.qgs", "project.qgs"),
            ],
            expected_files=[
                "data.gpkg",
                "project_qfield.qgs",
            ],
            tempdir=tempdir,
        )

        local_file = os.path.join(tempdir, "project_qfield.qgs")
        with open(local_file, "r") as f:
            self.assertEqual(
                f.readline().strip(),
                "<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>",
            )

    def test_list_files_for_qfield_broken_file(self):
        self.upload_files(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/broken.qgs", "broken.qgs"),
            ],
        )

        response = self.client.post(
            "/api/v1/jobs/",
            {
                "project_id": self.project1.id,
                "type": Job.Type.PACKAGE,
            },
        )

        self.assertTrue(status.is_success(response.status_code))
        job_id = response.json().get("id")

        # Wait for the worker to finish
        for _ in range(10):
            time.sleep(3)
            response = self.client.get(
                f"/api/v1/jobs/{job_id}/",
            )
            if response.json()["status"] == "failed":
                return

        self.fail("Worker didn't finish")

    def test_create_job_twice(self):
        self.upload_files(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/project2.qgs", "project.qgs"),
                ("delta/points.geojson", "points.geojson"),
            ],
        )

        response = self.client.post(
            "/api/v1/jobs/",
            {
                "project_id": self.project1.id,
                "type": Job.Type.PACKAGE,
            },
        )

        self.assertTrue(response.status_code, 201)

        response = self.client.post(
            "/api/v1/jobs/",
            {
                "project_id": self.project1.id,
                "type": Job.Type.PACKAGE,
            },
        )

        self.assertTrue(response.status_code, 200)

    def test_downloaded_file_has_canvas_name(self):
        tempdir = tempfile.mkdtemp()

        self.upload_files_and_check_package(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/nonspatial.csv", "nonspatial.csv"),
                ("delta/testdata.gpkg", "testdata.gpkg"),
                ("delta/points.geojson", "points.geojson"),
                ("delta/polygons.geojson", "polygons.geojson"),
                ("delta/project.qgs", "project.qgs"),
            ],
            expected_files=[
                "data.gpkg",
                "project_qfield.qgs",
            ],
            tempdir=tempdir,
        )

        local_file = os.path.join(tempdir, "project_qfield.qgs")
        with open(local_file, "r") as f:
            for line in f:
                if 'name="theMapCanvas"' in line:
                    return

    def test_download_project_with_broken_layer_datasources(self):
        self.upload_files_and_check_package(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/points.geojson", "points.geojson"),
                (
                    "delta/project_broken_datasource.qgs",
                    "project_broken_datasource.qgs",
                ),
            ],
            expected_files=[
                "data.gpkg",
                "project_broken_datasource_qfield.qgs",
            ],
            invalid_layers=["surfacestructure_35131bca_337c_483b_b09e_1cf77b1dfb16"],
        )

    def test_needs_repackaging_without_online_vector(self):
        self.project1.refresh_from_db()
        # newly uploaded project should always need to be packaged at least once
        self.assertTrue(self.project1.needs_repackaging)

        self.upload_files_and_check_package(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/nonspatial.csv", "nonspatial.csv"),
                ("delta/testdata.gpkg", "testdata.gpkg"),
                ("delta/points.geojson", "points.geojson"),
                ("delta/polygons.geojson", "polygons.geojson"),
                ("delta/project.qgs", "project.qgs"),
            ],
            expected_files=[
                "data.gpkg",
                "project_qfield.qgs",
            ],
        )

        self.project1.refresh_from_db()
        # no longer needs repackaging since geopackage layers cannot change without deltas/reupload
        self.assertFalse(self.project1.needs_repackaging)

        self.upload_files(
            self.token1.key,
            self.project1,
            files=[
                ("delta/nonspatial.csv", "nonspatial.csv"),
            ],
        )

        self.project1.refresh_from_db()
        # a layer file changed, so we need to repackage
        self.assertTrue(self.project1.needs_repackaging)

    def test_needs_repackaging_with_online_vector(self):
        cur = self.conn.cursor()
        cur.execute("CREATE TABLE point (id integer, geometry geometry(point, 2056))")
        self.conn.commit()
        cur.execute(
            "INSERT INTO point(id, geometry) VALUES(1, ST_GeomFromText('POINT(2725505 1121435)', 2056))"
        )
        self.conn.commit()

        self.project1.refresh_from_db()
        # newly uploaded project should always need to be packaged at least once
        self.assertTrue(self.project1.needs_repackaging)

        self.upload_files_and_check_package(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/project2.qgs", "project.qgs"),
                ("delta/points.geojson", "points.geojson"),
            ],
            expected_files=["data.gpkg", "project_qfield.qgs"],
        )

        self.project1.refresh_from_db()
        # projects with online vector layer should always show as it needs repackaging
        self.assertTrue(self.project1.needs_repackaging)

    def test_connects_via_pgservice(self):
        cur = self.conn.cursor()
        cur.execute("CREATE TABLE point (id integer, geometry geometry(point, 2056))")
        self.conn.commit()

        Secret.objects.create(
            name="PG_SERVICE_GEODB1",
            type=Secret.Type.PGSERVICE,
            project=self.project1,
            created_by=self.project1.owner,
            value=(
                "[geodb1]\n"
                "dbname=test\n"
                "host=geodb\n"
                "port=5432\n"
                f"user={os.environ.get('GEODB_USER')}\n"
                f"password={os.environ.get('GEODB_PASSWORD')}\n"
                "sslmode=disable\n"
            ),
        )

        Secret.objects.create(
            name="PG_SERVICE_GEODB2",
            type=Secret.Type.PGSERVICE,
            project=self.project1,
            created_by=self.project1.owner,
            value=(
                "[geodb2]\n"
                "dbname=test\n"
                "host=geodb\n"
                "port=5432\n"
                f"user={os.environ.get('GEODB_USER')}\n"
                f"password={os.environ.get('GEODB_PASSWORD')}\n"
                "sslmode=disable\n"
            ),
        )

        self.upload_files(
            self.token1.key,
            self.project1,
            files=[
                ("delta/project_pgservice.qgs", "project.qgs"),
            ],
        )

        self.wait_for_project_ok_status(self.project1)
        self.project1.refresh_from_db()

        last_process_job = Job.objects.filter(type=Job.Type.PROCESS_PROJECTFILE).latest(
            "updated_at"
        )
        layers_by_id = last_process_job.feedback["outputs"]["project_details"][
            "project_details"
        ]["layers_by_id"]

        self.assertEqual(last_process_job.status, Job.Status.FINISHED)
        self.assertTrue(
            layers_by_id["point_6b900fa7_af52_4082_bbff_6077f4a91d02"]["is_valid"]
        )

    def test_has_online_vector_data(self):
        cur = self.conn.cursor()
        cur.execute("CREATE TABLE point (id integer, geometry geometry(point, 2056))")
        self.conn.commit()

        self.upload_files(
            self.token1.key,
            self.project1,
            files=[
                ("delta/project2.qgs", "project.qgs"),
            ],
        )

        self.wait_for_project_ok_status(self.project1)

        self.project1.refresh_from_db()

        self.assertTrue(self.project1.has_online_vector_data)

    def test_has_no_online_vector_data(self):
        self.upload_files(
            self.token1.key,
            self.project1,
            files=[
                ("delta/project.qgs", "project.qgs"),
            ],
        )

        self.wait_for_project_ok_status(self.project1)

        self.project1.refresh_from_db()

        self.assertFalse(self.project1.has_online_vector_data)

    def test_filename_with_whitespace(self):
        self.upload_files_and_check_package(
            token=self.token1.key,
            project=self.project1,
            files=[
                ("delta/nonspatial.csv", "nonspatial.csv"),
                ("delta/testdata.gpkg", "testdata.gpkg"),
                ("delta/points.geojson", "points.geojson"),
                ("delta/polygons.geojson", "polygons.geojson"),
                ("delta/project.qgs", "project.qgs"),
            ],
            expected_files=[
                "data.gpkg",
                "project_qfield.qgs",
            ],
        )


# docker-app/qfieldcloud/core/tests/test_permission.py

import logging

from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.models import (
    Organization,
    OrganizationMember,
    Project,
    ProjectCollaborator,
    User,
)
from rest_framework import status
from rest_framework.test import APITestCase

from .utils import setup_subscription_plans, testdata_path

logging.disable(logging.CRITICAL)


class QfcTestCase(APITestCase):
    def setUp(self):
        setup_subscription_plans()

        # Create a user
        self.user1 = User.objects.create_user(username="user1", password="abc123")
        self.token1 = AuthToken.objects.get_or_create(user=self.user1)[0]

        # Create a second user
        self.user2 = User.objects.create_user(username="user2", password="abc123")
        self.token2 = AuthToken.objects.get_or_create(user=self.user2)[0]

        # Create an organization
        self.organization1 = Organization.objects.create(
            username="organization1",
            password="abc123",
            user_type=2,
            organization_owner=self.user1,
        )

        # Create a project
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )
        self.project1.save()

    def test_collaborator_project_takeover(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get("/api/v1/projects/", format="json")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(1, len(response.data))
        self.assertEqual("project1", response.data[0].get("name"))

        project = Project.objects.get(pk=response.data[0].get("id"))

        response = self.client.patch(
            f"/api/v1/projects/{project.pk}/",
            {"name": "renamed-project", "owner": "organization1"},
        )
        self.assertTrue(status.is_success(response.status_code))

        # user2 doesn't get to see user1's organization's project
        self.client.logout()

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token2.key)
        response = self.client.get("/api/v1/projects/", format="json")
        self.assertEqual(0, len(response.data))

        # user2 is added to the project and the organization
        ProjectCollaborator.objects.create(
            project=project,
            collaborator=self.user2,
            role=ProjectCollaborator.Roles.READER,
        )
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user2,
        )
        response = self.client.get("/api/v1/projects/", format="json")
        self.assertEqual(1, len(response.data))
        self.assertEqual("renamed-project", response.data[0].get("name"))

        # patch is denied
        response = self.client.patch(
            f"/api/v1/projects/{project.pk}/",
            {"name": "stolen-project", "owner": "user2"},
        )
        self.assertFalse(status.is_success(response.status_code))
        project.refresh_from_db()
        self.assertEqual("renamed-project", project.name)

    def test_reader_cannot_push(self):
        # Connect as user2
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token2.key)

        # Set user2 as a collaborator READER of project1
        self.collaborator1 = ProjectCollaborator.objects.create(
            project=self.project1,
            collaborator=self.user2,
            role=ProjectCollaborator.Roles.READER,
        )

        file_path = testdata_path("file.txt")
        # Push a file
        response = self.client.post(
            "/api/v1/files/{}/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)

    def test_reporter_can_push(self):
        # Connect as user2
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token2.key)

        # Set user2 as a collaborator REPORTER of project1, and as part of the org owning the project
        self.project1.owner = self.organization1
        self.project1.save()
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user2,
        )
        self.collaborator1 = ProjectCollaborator.objects.create(
            project=self.project1,
            collaborator=self.user2,
            role=ProjectCollaborator.Roles.REPORTER,
        )

        file_path = testdata_path("file.txt")
        # Push a file
        response = self.client.post(
            "/api/v1/files/{}/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

    def test_cannot_list_files_of_private_project(self):
        # Connect as user2
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token2.key)

        # List files
        response = self.client.get("/api/v1/files/{}/".format(self.project1.id))
        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)


# docker-app/qfieldcloud/core/tests/test_project.py

import logging

from django.core.exceptions import ValidationError
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.models import (
    Organization,
    OrganizationMember,
    Project,
    ProjectCollaborator,
    Team,
    TeamMember,
    User,
)
from rest_framework import status
from rest_framework.test import APITestCase

from .utils import setup_subscription_plans

logging.disable(logging.CRITICAL)


class QfcTestCase(APITestCase):
    def setUp(self):
        setup_subscription_plans()

        # Create a user
        self.user1 = User.objects.create_user(username="user1", password="abc123")
        self.token1 = AuthToken.objects.get_or_create(user=self.user1)[0]

        # Create a user
        self.user2 = User.objects.create_user(username="user2", password="abc123")
        self.token2 = AuthToken.objects.get_or_create(user=self.user2)[0]

        # Create a user
        self.user3 = User.objects.create_user(username="user3", password="abc123")
        self.token3 = AuthToken.objects.get_or_create(user=self.user3)[0]

    def test_create_project(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.post(
            "/api/v1/projects/",
            {
                "name": "api_created_project",
                "owner": "user1",
                "description": "desc",
                "is_public": False,
            },
        )
        self.assertTrue(status.is_success(response.status_code))

        project = Project.objects.get(name="api_created_project")
        # Will raise exception if it doesn't exist

        self.assertEqual(str(project.owner), "user1")

    def test_create_project_invalid_name(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.post(
            "/api/v1/projects/",
            {
                "name": "project$",
                "owner": "user1",
                "description": "desc",
                "is_public": False,
            },
        )
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    def test_list_public_projects(self):

        # Create a public project of user2
        self.project1 = Project.objects.create(
            name="project1", is_public=True, owner=self.user2
        )

        # Create a private project of user2
        self.project1 = Project.objects.create(
            name="project2", is_public=False, owner=self.user2
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # 1) do not list the public projects by default
        response = self.client.get("/api/v1/projects/")
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.data), 0)

        # 1) list the public projects on request
        response = self.client.get("/api/v1/projects/?include-public=1")
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.data), 1)
        self.assertEqual(response.data[0]["name"], "project1")
        self.assertEqual(response.data[0]["owner"], "user2")

        # 1) do not list the public projects for any other value than 1
        response = self.client.get("/api/v1/projects/?include-public=true")
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.data), 0)

    def test_list_collaborators_of_project(self):

        # Create a project of user1
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        # Add user2 as collaborator
        ProjectCollaborator.objects.create(
            project=self.project1,
            collaborator=self.user2,
            role=ProjectCollaborator.Roles.MANAGER,
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.get("/api/v1/collaborators/{}/".format(self.project1.id))

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.data), 1)

        json = response.json()
        json = sorted(json, key=lambda k: k["collaborator"])

        self.assertEqual(json[0]["collaborator"], "user2")
        self.assertEqual(json[0]["role"], "manager")

    def test_list_projects_of_authenticated_user(self):

        # Create a project of user1
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        # Create another project of user1
        self.project2 = Project.objects.create(
            name="project2", is_public=False, owner=self.user1
        )

        # Create a project of user2 without access to user1
        self.project3 = Project.objects.create(
            name="project3", is_public=False, owner=self.user2
        )

        # Create a project of user2 with invalid access to user1
        # (invalid because project is not owned by an organization the user is part of)
        self.project4 = Project.objects.create(
            name="project4", is_public=False, owner=self.user2
        )

        ProjectCollaborator.objects.create(
            project=self.project4,
            collaborator=self.user1,
            role=ProjectCollaborator.Roles.MANAGER,
        )

        # Create a project of user2 with valid access to user1
        org1 = Organization.objects.create(
            username="org1", organization_owner=self.user2
        )
        self.project5 = Project.objects.create(
            name="project5", is_public=False, owner=org1
        )

        OrganizationMember.objects.create(
            organization=org1,
            member=self.user1,
        )
        ProjectCollaborator.objects.create(
            project=self.project5,
            collaborator=self.user1,
            role=ProjectCollaborator.Roles.MANAGER,
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.get("/api/v1/projects/")

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.data), 3)

        json = response.json()
        json = sorted(json, key=lambda k: k["name"])

        self.assertEqual(json[0]["name"], "project1")
        self.assertEqual(json[0]["owner"], "user1")
        self.assertEqual(json[0]["user_role"], "admin")
        self.assertEqual(json[0]["user_role_origin"], "project_owner")
        self.assertEqual(json[1]["name"], "project2")
        self.assertEqual(json[1]["owner"], "user1")
        self.assertEqual(json[1]["user_role"], "admin")
        self.assertEqual(json[1]["user_role_origin"], "project_owner")
        self.assertEqual(json[2]["name"], "project5")
        self.assertEqual(json[2]["owner"], "org1")
        self.assertEqual(json[2]["user_role"], "manager")
        self.assertEqual(json[2]["user_role_origin"], "collaborator")

    def test_create_collaborator(self):

        # Create a project of user1
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.post(
            "/api/v1/collaborators/{}/".format(self.project1.id),
            {
                "collaborator": "user2",
                "role": "editor",
            },
        )
        self.assertTrue(status.is_success(response.status_code))

        collaborators = ProjectCollaborator.objects.all()
        self.assertEqual(len(collaborators), 1)
        self.assertEqual(collaborators[0].project, self.project1)
        self.assertEqual(collaborators[0].collaborator, self.user2)
        self.assertEqual(collaborators[0].role, ProjectCollaborator.Roles.EDITOR)

    def test_get_collaborator(self):

        # Create a project of user1
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        # Add user2 as collaborator
        ProjectCollaborator.objects.create(
            project=self.project1,
            collaborator=self.user2,
            role=ProjectCollaborator.Roles.REPORTER,
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.get(
            "/api/v1/collaborators/{}/user2/".format(self.project1.id)
        )

        self.assertTrue(status.is_success(response.status_code))
        json = response.json()
        self.assertEqual(json["collaborator"], "user2")
        self.assertEqual(json["role"], "reporter")

    def test_update_collaborator(self):

        # Create a project of user1
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        # Add user2 as collaborator
        ProjectCollaborator.objects.create(
            project=self.project1,
            collaborator=self.user2,
            role=ProjectCollaborator.Roles.REPORTER,
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.patch(
            "/api/v1/collaborators/{}/user2/".format(self.project1.id),
            {
                "role": "admin",
            },
        )

        self.assertTrue(status.is_success(response.status_code))

        collaborators = ProjectCollaborator.objects.all()
        self.assertEqual(len(collaborators), 1)
        self.assertEqual(collaborators[0].project, self.project1)
        self.assertEqual(collaborators[0].collaborator, self.user2)
        self.assertEqual(collaborators[0].role, ProjectCollaborator.Roles.ADMIN)

    def test_delete_collaborator(self):

        # Create a project of user1
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        # Add user2 as collaborator
        ProjectCollaborator.objects.create(
            project=self.project1,
            collaborator=self.user2,
            role=ProjectCollaborator.Roles.REPORTER,
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.delete(
            "/api/v1/collaborators/{}/user2/".format(self.project1.id)
        )

        self.assertTrue(status.is_success(response.status_code))

        collaborators = ProjectCollaborator.objects.all()
        self.assertEqual(len(collaborators), 0)

    def test_delete_project(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Create a project of user1
        project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        # Delete the project
        response = self.client.delete("/api/v1/projects/{}/".format(project1.id))
        self.assertTrue(status.is_success(response.status_code))

        # The project should not exist anymore
        self.assertFalse(Project.objects.filter(id=project1.id).exists())

    def test_error_responses(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Create a project of user1
        project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user2
        )

        # Get the project without permissions
        response = self.client.get("/api/v1/projects/{}/".format(project1.id))

        self.assertEqual(response.status_code, 403)
        self.assertEqual(response.json()["code"], "api_error")

        # Get an unexisting project id
        response = self.client.get(
            "/api/v1/projects/{}/".format("a258db08-b1cb-4c34-a0cc-1e0e2a464f87")
        )

        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.json()["code"], "object_not_found")

        # Get a project without a valid project id
        response = self.client.get("/api/v1/projects/{}/".format("pizza"))
        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.json()["code"], "validation_error")

    def test_public_projecs(self):
        # Create a project of user1
        self.project1 = Project.objects.create(
            name="project1", is_public=True, owner=self.user1
        )

        # Create another project of user2
        self.project2 = Project.objects.create(
            name="project2", is_public=True, owner=self.user2
        )

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.get("/api/v1/projects/public", follow=True)

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.data), 2)

        json = response.json()
        json = sorted(json, key=lambda k: k["name"])

        self.assertEqual(json[0]["name"], "project1")
        self.assertEqual(json[0]["owner"], "user1")
        self.assertEqual(json[0]["user_role"], "admin")
        self.assertEqual(json[0]["user_role_origin"], "project_owner")
        self.assertEqual(json[1]["name"], "project2")
        self.assertEqual(json[1]["owner"], "user2")
        self.assertEqual(json[1]["user_role"], "reader")
        self.assertEqual(json[1]["user_role_origin"], "public")

    def test_private_project_memberships(self):
        """Tests for QF-1553 - limit collaboration on private projects"""

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Create a project with a collaborator
        u = User.objects.create(username="u")
        o = Organization.objects.create(username="o", organization_owner=u)
        p = Project.objects.create(name="p", owner=u, is_public=True)

        apiurl = f"/api/v1/projects/{p.pk}/"

        self.client.raise_request_exception = True

        def assert_no_role():
            response = self.client.get(apiurl, follow=True)
            self.assertEqual(response.status_code, 403)

        def assert_role(role, origin):
            response = self.client.get(apiurl, follow=True)
            self.assertEqual(response.status_code, 200)
            json = response.json()
            self.assertEqual(json["user_role"], role)
            self.assertEqual(json["user_role_origin"], origin)

        # Project is public, we have a public role
        assert_role("reader", "public")

        # Project is public, collaboration membership is valid
        ProjectCollaborator.objects.create(
            project=p, collaborator=self.user1, role=ProjectCollaborator.Roles.MANAGER
        )
        assert_role("manager", "collaborator")

        # If project is made private, the collaboration is invalid
        p.is_public = False
        p.save()
        assert_no_role()

        # Making the owner an organisation is not enough as user is not member of that org
        p.owner = o
        p.save()
        assert_no_role()

        # As the user must be member of the organisation
        OrganizationMember.objects.create(organization=o, member=self.user1)
        assert_role("manager", "collaborator")

    def test_add_project_collaborator_without_being_org_member(self):
        u1 = User.objects.create(username="u1")
        u2 = User.objects.create(username="u2")
        o1 = Organization.objects.create(username="o1", organization_owner=u1)
        p1 = Project.objects.create(name="p1", owner=o1, is_public=False)

        # cannot add project collaborator if not already an org member
        with self.assertRaises(ValidationError):
            ProjectCollaborator.objects.create(
                project=p1, collaborator=u2, role=ProjectCollaborator.Roles.MANAGER
            )

    def test_add_project_collaborator_and_being_org_member(self):
        u1 = User.objects.create(username="u1")
        u2 = User.objects.create(username="u2")
        o1 = Organization.objects.create(username="o1", organization_owner=u1)
        p1 = Project.objects.create(name="p1", owner=o1, is_public=False)

        # add project collaborator if the user is already an organization member
        OrganizationMember.objects.create(organization=o1, member=u2)
        ProjectCollaborator.objects.create(
            project=p1, collaborator=u2, role=ProjectCollaborator.Roles.MANAGER
        )

    def test_add_team_member_without_being_org_member(self):
        u1 = User.objects.create(username="u1")
        u2 = User.objects.create(username="u2")
        o1 = Organization.objects.create(username="o1", organization_owner=u1)
        t1 = Team.objects.create(username="t1", team_organization=o1)

        # cannot add team member if not already an org members
        with self.assertRaises(ValidationError):
            TeamMember.objects.create(team=t1, member=u2)

    def test_add_team_member_and_being_org_member(self):
        u1 = User.objects.create(username="u1")
        u2 = User.objects.create(username="u2")
        o1 = Organization.objects.create(username="o1", organization_owner=u1)
        t1 = Team.objects.create(username="t1", team_organization=o1)

        # add team member if not already an org members
        OrganizationMember.objects.create(organization=o1, member=u2)
        TeamMember.objects.create(team=t1, member=u2)

    def test_delete_org_member_should_delete_team_membership(self):
        u1 = User.objects.create(username="u1")
        u2 = User.objects.create(username="u2")
        o1 = Organization.objects.create(username="o1", organization_owner=u1)
        t1 = Team.objects.create(username="t1", team_organization=o1)

        # deleting an organization member via queryset should also delete team membership
        OrganizationMember.objects.create(organization=o1, member=u2)
        TeamMember.objects.create(team=t1, member=u2)

        self.assertEqual(TeamMember.objects.filter(team=t1, member=u2).count(), 1)

        OrganizationMember.objects.filter(organization=o1, member=u2).delete()

        self.assertEqual(TeamMember.objects.filter(team=t1, member=u2).count(), 0)

        # deleting an organization member via model should also delete team membership
        om1 = OrganizationMember.objects.create(organization=o1, member=u2)
        TeamMember.objects.create(team=t1, member=u2)

        self.assertEqual(TeamMember.objects.filter(team=t1, member=u2).count(), 1)

        om1.delete()

        self.assertEqual(TeamMember.objects.filter(team=t1, member=u2).count(), 0)


# docker-app/qfieldcloud/core/tests/test_qfield_file.py

import json
import logging
import os
import tempfile
import time

import psycopg2
import requests
from django.http.response import HttpResponse, HttpResponseRedirect
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.geodb_utils import delete_db_and_role
from qfieldcloud.core.models import Geodb, Job, PackageJob, Project, User
from rest_framework import status
from rest_framework.test import APITransactionTestCase

from .utils import setup_subscription_plans, testdata_path

logging.disable(logging.CRITICAL)


class QfcTestCase(APITransactionTestCase):
    def setUp(self):
        setup_subscription_plans()

        # Create a user
        self.user1 = User.objects.create_user(username="user1", password="abc123")

        self.user2 = User.objects.create_user(username="user2", password="abc123")

        self.token1 = AuthToken.objects.get_or_create(user=self.user1)[0]

        # Create a project
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        try:
            delete_db_and_role("test", self.user1.username)
        except Exception:
            pass

        self.geodb = Geodb.objects.create(
            user=self.user1,
            dbname="test",
            hostname="geodb",
            port=5432,
        )

        self.conn = psycopg2.connect(
            dbname="test",
            user=os.environ.get("GEODB_USER"),
            password=os.environ.get("GEODB_PASSWORD"),
            host="geodb",
            port=5432,
        )

    def tearDown(self):
        self.conn.close()

    def fail(self, msg: str, job: Job = None):
        if job:
            msg += f"\n\nOutput:\n================\n{job.output}\n================"

            if job.feedback:
                if "error_stack" in job.feedback:
                    msg += "\n\nError:\n================"
                    for single_error_stack in job.feedback["error_stack"]:
                        msg += "\n"
                        msg += single_error_stack

                    msg += f"  {job.feedback['error']}\n================"

                feedback = json.dumps(job.feedback, indent=2, sort_keys=True)
                msg += f"\n\nFeedback:\n================\n{feedback}\n================"
            else:
                msg += "\n\nFeedback: None"

        super().fail(msg)

    def assertHttpOk(self, response: HttpResponse):
        try:
            self.assertTrue(status.is_success(response.status_code), response.json())
        except Exception:
            self.assertTrue(status.is_success(response.status_code), response.content)

    def test_list_files_for_qfield(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        cur = self.conn.cursor()

        cur.execute(
            """
            CREATE TABLE point (
                id          integer,
                geometry   geometry(point, 2056)
            );
            """
        )

        self.conn.commit()

        cur.execute(
            """
            INSERT INTO point(id, geometry)
            VALUES(1, ST_GeomFromText('POINT(2725505 1121435)', 2056));
            """
        )
        self.conn.commit()

        # Add the qgis project
        file = testdata_path("delta/project2.qgs")
        response = self.client.post(
            "/api/v1/files/{}/project.qgs/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        response = self.client.post(
            "/api/v1/qfield-files/export/{}/".format(self.project1.id)
        )
        self.assertTrue(status.is_success(response.status_code))

        # Wait for the worker to finish
        for _ in range(20):
            time.sleep(3)
            response = self.client.get(
                "/api/v1/qfield-files/export/{}/".format(self.project1.id),
            )
            payload = response.json()
            if payload["status"] == "STATUS_EXPORTED":
                response = self.client.get(
                    "/api/v1/qfield-files/{}/".format(self.project1.id),
                )
                json_resp = response.json()
                files = sorted(json_resp["files"], key=lambda k: k["name"])
                self.assertEqual(files[0]["name"], "data.gpkg")
                self.assertEqual(files[1]["name"], "project_qfield.qgs")
                return
            elif payload["status"] == "STATUS_ERROR":
                self.fail("Worker failed with error")

        self.fail("Worker didn't finish")

    def test_list_files_for_qfield_incomplete_project(self):
        # the qgs file is missing
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Add files to the project
        file = testdata_path("delta/points.geojson")
        response = self.client.post(
            "/api/v1/files/{}/points.geojson/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        response = self.client.post(
            "/api/v1/qfield-files/export/{}/".format(self.project1.id)
        )
        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.json()["code"], "no_qgis_project")

    def test_download_file_for_qfield(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Add files to the project
        file = testdata_path("delta/points.geojson")
        response = self.client.post(
            "/api/v1/files/{}/points.geojson/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        file = testdata_path("delta/polygons.geojson")
        response = self.client.post(
            "/api/v1/files/{}/polygons.geojson/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        file = testdata_path("delta/project.qgs")
        response = self.client.post(
            "/api/v1/files/{}/project.qgs/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        # Launch the export
        response = self.client.post(
            "/api/v1/qfield-files/export/{}/".format(self.project1.id),
        )
        self.assertTrue(status.is_success(response.status_code))

        # Wait for the worker to finish
        for _ in range(10):
            time.sleep(3)
            response = self.client.get(
                "/api/v1/qfield-files/export/{}/".format(self.project1.id),
            )

            self.assertHttpOk(response)
            payload = response.json()

            if payload["status"] == "STATUS_EXPORTED":
                response = self.client.get(
                    f"/api/v1/qfield-files/{self.project1.id}/project_qfield.qgs/"
                )

                self.assertIsInstance(response, HttpResponseRedirect)

                temp_dir = tempfile.mkdtemp()
                local_file = os.path.join(temp_dir, "project.qgs")

                # We cannot use the self.client HTTP client, since it does not support
                # requests outside the current Django App
                # Using the rest_api_framework.RequestsClient is not much better, so better
                # use the `requests` module

                with requests.get(response.url, stream=True) as r:
                    with open(local_file, "wb") as f:
                        for chunk in r.iter_content():
                            f.write(chunk)

                with open(local_file, "r") as f:
                    self.assertEqual(
                        f.readline().strip(),
                        "<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>",
                    )
                return
            elif payload["status"] == "STATUS_ERROR":
                self.fail(
                    "Worker failed with error",
                    job=PackageJob.objects.filter(project=self.project1).last(),
                )

        self.fail("Worker didn't finish")

    def test_list_files_for_qfield_broken_file(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Add files to the project
        file = testdata_path("delta/broken.qgs")
        response = self.client.post(
            "/api/v1/files/{}/broken.qgs/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        # Launch the export
        response = self.client.post(
            "/api/v1/qfield-files/export/{}/".format(self.project1.id),
        )
        self.assertTrue(status.is_success(response.status_code))

        # Wait for the worker to finish
        for _ in range(10):
            time.sleep(3)
            response = self.client.get(
                "/api/v1/qfield-files/export/{}/".format(self.project1.id),
            )

            self.assertHttpOk(response)

            if response.json()["status"] == "STATUS_ERROR":
                return

        self.fail(
            "Worker didn't finish", job=Job.objects.filter(project=self.project1).last()
        )

    def test_downloaded_file_has_canvas_name(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Add files to the project
        file = testdata_path("delta/points.geojson")
        response = self.client.post(
            "/api/v1/files/{}/points.geojson/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        file = testdata_path("delta/polygons.geojson")
        response = self.client.post(
            "/api/v1/files/{}/polygons.geojson/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        file = testdata_path("delta/project.qgs")
        response = self.client.post(
            "/api/v1/files/{}/project.qgs/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        # Launch the export
        response = self.client.post(
            "/api/v1/qfield-files/export/{}/".format(self.project1.id),
        )
        self.assertTrue(status.is_success(response.status_code))

        # Wait for the worker to finish
        for _ in range(10):
            time.sleep(3)
            response = self.client.get(
                "/api/v1/qfield-files/export/{}/".format(self.project1.id),
            )

            payload = response.json()
            if payload["status"] == "STATUS_EXPORTED":
                response = self.client.get(
                    f"/api/v1/qfield-files/{self.project1.id}/project_qfield.qgs/"
                )

                self.assertIsInstance(response, HttpResponseRedirect)

                temp_dir = tempfile.mkdtemp()
                local_file = os.path.join(temp_dir, "project.qgs")

                # We cannot use the self.client HTTP client, since it does not support
                # requests outside the current Django App
                # Using the rest_api_framework.RequestsClient is not much better, so better
                # use the `requests` module
                with requests.get(response.url, stream=True) as r:
                    with open(local_file, "wb") as f:
                        for chunk in r.iter_content():
                            f.write(chunk)

                with open(local_file, "r") as f:
                    for line in f:
                        if 'name="theMapCanvas"' in line:
                            return
                self.fail(
                    'Worker failed, missing .qgs XML attribute: name="theMapCanvas"'
                )
            elif payload["status"] == "STATUS_ERROR":
                self.fail("Worker failed with error")

        self.fail("Worker didn't finish or there was an error")

    def test_download_project_with_broken_layer_datasources(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Add files to the project
        file = testdata_path("delta/points.geojson")
        response = self.client.post(
            "/api/v1/files/{}/points.geojson/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        file = testdata_path("delta/project_broken_datasource.qgs")
        response = self.client.post(
            "/api/v1/files/{}/project.qgs/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        # Launch the export
        response = self.client.post(
            "/api/v1/qfield-files/export/{}/".format(self.project1.id),
        )
        self.assertTrue(status.is_success(response.status_code))

        # Wait for the worker to finish
        for _ in range(10):
            time.sleep(3)
            response = self.client.get(
                "/api/v1/qfield-files/export/{}/".format(self.project1.id),
            )
            payload = response.json()
            if payload["status"] == "STATUS_EXPORTED":

                response = self.client.get(
                    "/api/v1/qfield-files/{}/".format(self.project1.id),
                )

                self.assertHttpOk(response)

                export_payload = response.json()
                layer_ok = export_payload["layers"][
                    "points_c2784cf9_c9c3_45f6_9ce5_98a6047e4d6c"
                ]
                layer_failed = export_payload["layers"][
                    "surfacestructure_35131bca_337c_483b_b09e_1cf77b1dfb16"
                ]

                self.assertTrue(layer_ok["valid"], layer_ok["status"])
                self.assertFalse(layer_failed["valid"], layer_failed["status"])
                return
            elif payload["status"] == "STATUS_ERROR":
                self.fail(
                    "Worker failed with error",
                    job=Job.objects.filter(project=self.project1).last(),
                )

        self.fail("Worker didn't finish")

    def test_filename_with_whitespace(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        # Add files to the project
        file = testdata_path("delta/points.geojson")
        response = self.client.post(
            "/api/v1/files/{}/points.geojson/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        file = testdata_path("delta/polygons.geojson")
        response = self.client.post(
            "/api/v1/files/{}/polygons.geojson/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        file = testdata_path("delta/project.qgs")
        response = self.client.post(
            "/api/v1/files/{}/whitespace project.qgs/".format(self.project1.id),
            {"file": open(file, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))

        # Launch the export
        response = self.client.post(
            "/api/v1/qfield-files/export/{}/".format(self.project1.id),
        )
        self.assertTrue(status.is_success(response.status_code))

        # Wait for the worker to finish
        for _ in range(10):
            time.sleep(3)
            response = self.client.get(
                "/api/v1/qfield-files/export/{}/".format(self.project1.id),
            )

            payload = response.json()
            if payload["status"] == "STATUS_EXPORTED":
                return
            elif payload["status"] == "STATUS_ERROR":
                self.fail("Worker failed with error")

        self.fail("Worker didn't finish or there was an error")


# docker-app/qfieldcloud/core/tests/test_qgis_file.py

import io
import logging
import tempfile
import time
from pathlib import PurePath

from django.core.management import call_command
from django.http import FileResponse
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core import utils
from qfieldcloud.core.models import Job, ProcessProjectfileJob, Project, User
from qfieldcloud.subscription.models import Plan
from rest_framework import status
from rest_framework.test import APITransactionTestCase

from .utils import get_filename, setup_subscription_plans, testdata_path

logging.disable(logging.CRITICAL)


class QfcTestCase(APITransactionTestCase):
    def setUp(self):
        setup_subscription_plans()

        # Create a user
        self.user1 = User.objects.create_user(username="user1", password="abc123")
        self.user1.save()

        self.user2 = User.objects.create_user(username="user2", password="abc123")
        self.user2.save()

        self.token1 = AuthToken.objects.get_or_create(user=self.user1)[0]

        # Create a project
        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )
        self.project1.save()

    def get_file_contents(self, project, filename, version=None):
        qs = ""
        if version:
            qs = f"?version={version}"

        response = self.client.get(f"/api/v1/files/{project.id}/{filename}/{qs}")

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(get_filename(response), PurePath(filename).name)

        if isinstance(response, FileResponse):
            return b"".join(response.streaming_content)
        else:
            return response.content

    def test_push_file(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        # Push a file
        response = self.client.post(
            "/api/v1/files/{}/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

    def test_push_download_file(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        # Push a file
        response = self.client.post(
            "/api/v1/files/{}/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        self.assertEqual(
            self.get_file_contents(self.project1, "file.txt"),
            open(testdata_path("file.txt"), "rb").read(),
        )

    def test_push_download_file_with_path(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        # Push a file
        response = self.client.post(
            "/api/v1/files/{}/foo/bar/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        # Pull the file
        response = self.client.get(
            f"/api/v1/files/{self.project1.id}/foo/bar/file.txt/",
            stream=True,
        )

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(get_filename(response), "file.txt")

        self.assertEqual(
            self.get_file_contents(self.project1, "foo/bar/file.txt"),
            open(testdata_path("file.txt"), "rb").read(),
        )

    def test_push_list_file(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        # Push a file
        response = self.client.post(
            "/api/v1/files/{}/aaa/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        file_path = testdata_path("file2.txt")
        # Push a second file
        response = self.client.post(
            "/api/v1/files/{}/file2.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 2)

        # List files
        response = self.client.get("/api/v1/files/{}/".format(self.project1.id))
        self.assertTrue(status.is_success(response.status_code))

        json = response.json()
        json = sorted(json, key=lambda k: k["name"])

        self.assertEqual(json[0]["name"], "aaa/file.txt")
        self.assertEqual(json[0]["size"], 13)
        self.assertEqual(json[1]["name"], "file2.txt")
        self.assertEqual(json[1]["size"], 13)
        self.assertEqual(
            json[0]["sha256"],
            "8663bab6d124806b9727f89bb4ab9db4cbcc3862f6bbf22024dfa7212aa4ab7d",
        )
        self.assertEqual(
            json[1]["sha256"],
            "fcc85fb502bd772aa675a0263b5fa665bccd5d8d93349d1dbc9f0f6394dd37b9",
        )

    def test_push_list_file_with_space_in_name(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        # Push a file
        project_file = "aaa bbb/project qgis 1.2.qgs"
        response = self.client.post(
            f"/api/v1/files/{self.project1.id}/{project_file}/",
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, project_file
        )

        # List files
        response = self.client.get("/api/v1/files/{}/".format(self.project1.id))
        self.assertTrue(status.is_success(response.status_code))

        json = response.json()

        self.assertEqual(json[0]["name"], "aaa bbb/project qgis 1.2.qgs")

    def test_push_list_file_versions(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        # Push a file
        response = self.client.post(
            "/api/v1/files/{}/aaa/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        # Wait 2 seconds to be sure the file timestamps are different
        time.sleep(2)

        file_path = testdata_path("file2.txt")
        # Push another file with the same name (i.e. push another
        # version)
        response = self.client.post(
            "/api/v1/files/{}/aaa/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        # List files
        response = self.client.get("/api/v1/files/{}/".format(self.project1.id))
        self.assertTrue(status.is_success(response.status_code))

        versions = sorted(
            response.json()[0]["versions"], key=lambda k: k["last_modified"]
        )

        self.assertEqual(len(versions), 2)
        self.assertNotEqual(versions[0]["last_modified"], versions[1]["last_modified"])

        self.assertEqual(
            versions[0]["sha256"],
            "8663bab6d124806b9727f89bb4ab9db4cbcc3862f6bbf22024dfa7212aa4ab7d",
        )
        self.assertEqual(
            versions[1]["sha256"],
            "fcc85fb502bd772aa675a0263b5fa665bccd5d8d93349d1dbc9f0f6394dd37b9",
        )

        self.assertEqual(versions[0]["size"], 13)
        self.assertEqual(versions[1]["size"], 13)

    def test_push_download_specific_version_file(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        # Push a file
        response = self.client.post(
            "/api/v1/files/{}/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        # Wait 2 seconds to be sure the file timestamps are different
        time.sleep(2)

        file_path = testdata_path("file2.txt")
        # Push another file with the same name (i.e. push another
        # version)
        response = self.client.post(
            "/api/v1/files/{}/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        self.assertNotEqual(
            self.get_file_contents(self.project1, "file.txt"),
            open(testdata_path("file.txt"), "rb").read(),
        )

        self.assertEqual(
            self.get_file_contents(self.project1, "file.txt"),
            open(testdata_path("file2.txt"), "rb").read(),
        )

        # List files
        response = self.client.get("/api/v1/files/{}/".format(self.project1.id))
        self.assertTrue(status.is_success(response.status_code))

        versions = sorted(
            response.json()[0]["versions"], key=lambda k: k["last_modified"]
        )

        # Pull the oldest version
        self.assertEqual(
            self.get_file_contents(
                self.project1, "file.txt", versions[0]["version_id"]
            ),
            open(testdata_path("file.txt"), "rb").read(),
        )

        # Pull the newest version
        self.assertEqual(
            self.get_file_contents(
                self.project1, "file.txt", versions[1]["version_id"]
            ),
            open(testdata_path("file2.txt"), "rb").read(),
        )

    def test_push_delete_file(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        # Push a file
        response = self.client.post(
            "/api/v1/files/{}/aaa/file.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        file_path = testdata_path("file2.txt")
        # Push a second file
        response = self.client.post(
            "/api/v1/files/{}/file2.txt/".format(self.project1.id),
            {"file": open(file_path, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 2)

        # List files
        response = self.client.get("/api/v1/files/{}/".format(self.project1.id))
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.json()), 2)

        # Delete a file
        response = self.client.delete(
            "/api/v1/files/{}/aaa/file.txt/".format(self.project1.id)
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        # List files
        response = self.client.get("/api/v1/files/{}/".format(self.project1.id))
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.json()), 1)

    def test_one_qgis_project_per_project(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        qgis_project_file = "foo/bar/file.qgs"
        # Push a QGIS project file
        response = self.client.post(
            f"/api/v1/files/{self.project1.id}/{qgis_project_file}/",
            {
                "file": open(file_path, "rb"),
            },
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename,
            qgis_project_file,
        )

        # Push again the same QGIS project file (this is allowed)
        response = self.client.post(
            f"/api/v1/files/{self.project1.id}/{qgis_project_file}/",
            {
                "file": open(file_path, "rb"),
            },
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename,
            qgis_project_file,
        )

        failing_qgis_project_file = "foo/bar/file2.qgs"
        # Push another QGIS project file
        response = self.client.post(
            f"/api/v1/files/{self.project1.id}/{failing_qgis_project_file}/",
            {
                "file": open(file_path, "rb"),
            },
            format="multipart",
        )
        self.assertEqual(response.status_code, 400)
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename,
            qgis_project_file,
        )

        # Push another QGIS project file
        response = self.client.post(
            "/api/v1/files/{}/foo/bar/file2.qgz/".format(self.project1.id),
            {
                "file": open(file_path, "rb"),
            },
            format="multipart",
        )
        self.assertEqual(response.status_code, 400)
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename,
            qgis_project_file,
        )

        response = self.client.delete(
            f"/api/v1/files/{self.project1.id}/{qgis_project_file}/",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

    def test_upload_1mb_file(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        big_file = tempfile.NamedTemporaryFile()
        with open(big_file.name, "wb") as bf:
            bf.truncate(1024 * 1024 * 1)

        # Push the file
        response = self.client.post(
            "/api/v1/files/{}/bigfile.big/".format(self.project1.id),
            data={"file": open(big_file.name, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        # List files
        response = self.client.get("/api/v1/files/{}/".format(self.project1.id))

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.json()), 1)
        self.assertEqual("bigfile.big", response.json()[0]["name"])
        self.assertGreater(response.json()[0]["size"], 1000000)
        self.assertLess(response.json()[0]["size"], 1100000)

    def test_upload_10mb_file(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        big_file = tempfile.NamedTemporaryFile()
        with open(big_file.name, "wb") as bf:
            bf.truncate(1024 * 1024 * 10)

        # Push the file
        response = self.client.post(
            "/api/v1/files/{}/bigfile.big/".format(self.project1.id),
            data={"file": open(big_file.name, "rb")},
            format="multipart",
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)

        # List files
        response = self.client.get("/api/v1/files/{}/".format(self.project1.id))

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(len(response.json()), 1)
        self.assertEqual("bigfile.big", response.json()[0]["name"])
        self.assertGreater(response.json()[0]["size"], 10000000)
        self.assertLess(response.json()[0]["size"], 11000000)

    def test_purge_old_versions_command(self):
        """This tests manual purging of old versions with the management command"""

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        acctype_3 = Plan.objects.create(storage_keep_versions=3, code="acc3")
        self.user1.useraccount.plan = acctype_3
        self.user1.useraccount.save()

        def count_versions():
            """counts the versions in first file of project1"""
            file = Project.objects.get(pk=self.project1.pk).files[0]
            return len(file.versions)

        def read_version(n):
            """returns the content of version in first file of project1"""
            file = Project.objects.get(pk=self.project1.pk).files[0]
            return file.versions[n]._data.get()["Body"].read().decode()

        # Create 20 versions (direct upload to s3)
        bucket = utils.get_s3_bucket()
        key = f"projects/{self.project1.id}/files/file.txt/"
        for i in range(20):
            test_file = io.BytesIO(f"v{i}".encode())
            bucket.upload_fileobj(test_file, key)

        # Ensure it worked
        self.assertEqual(count_versions(), 20)
        self.assertEqual(read_version(0), "v0")
        self.assertEqual(read_version(19), "v19")

        # Run management command on other project should have no effect
        other = Project.objects.create(name="other", owner=self.user1)
        call_command("purge_old_file_versions", "--force", "--projects", other.pk)
        self.assertEqual(count_versions(), 20)

        # Run management command should leave 3
        call_command("purge_old_file_versions", "--force")
        self.assertEqual(count_versions(), 3)
        self.assertEqual(read_version(0), "v17")
        self.assertEqual(read_version(2), "v19")

        # Run management command is idempotent
        call_command("purge_old_file_versions", "--force")
        self.assertEqual(count_versions(), 3)
        self.assertEqual(read_version(0), "v17")
        self.assertEqual(read_version(2), "v19")

    def test_purge_old_versions(self):
        """This tests automated purging of old versions when uploading files"""

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        apipath = f"/api/v1/files/{self.project1.id}/file.txt/"

        def count_versions():
            """counts the versions in first file of project1"""
            file = Project.objects.get(pk=self.project1.pk).files[0]
            return len(file.versions)

        def read_version(n):
            """returns the content of version in first file of project1"""
            file = Project.objects.get(pk=self.project1.pk).files[0]
            return file.versions[n]._data.get()["Body"].read().decode()

        # As PRO account, 10 version should be kept out of 20
        acctype_10 = Plan.objects.create(storage_keep_versions=10, code="acc10")
        self.user1.useraccount.plan = acctype_10
        self.user1.useraccount.save()
        for i in range(20):
            test_file = io.StringIO(f"v{i}")
            self.client.post(apipath, {"file": test_file}, format="multipart")
        self.assertEqual(count_versions(), 10)
        self.assertEqual(read_version(0), "v10")
        self.assertEqual(read_version(9), "v19")

        # As COMMUNITY account, 3 version should be kept
        acctype_3 = Plan.objects.create(storage_keep_versions=3, code="acc3")
        self.user1.useraccount.plan = acctype_3
        self.user1.useraccount.save()

        # But first we check that uploading to another project doesn't affect a projct
        otherproj = Project.objects.create(name="other", owner=self.user1)
        otherpath = f"/api/v1/files/{otherproj.id}/file.txt/"
        self.client.post(otherpath, {"file": io.StringIO("v1")}, format="multipart")
        self.assertEqual(count_versions(), 10)
        self.assertEqual(read_version(0), "v10")
        self.assertEqual(read_version(9), "v19")

        # As COMMUNITY account, 3 version should be kept out of 20 new ones
        for i in range(20, 40):
            test_file = io.StringIO(f"v{i}")
            self.client.post(apipath, {"file": test_file}, format="multipart")
        self.assertEqual(count_versions(), 3)
        self.assertEqual(read_version(0), "v37")
        self.assertEqual(read_version(2), "v39")

    def test_multiple_file_uploads_one_process_job(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
        self.assertEqual(
            Project.objects.get(pk=self.project1.pk).project_filename, None
        )

        file_path = testdata_path("file.txt")
        qgis_project_file = "foo/bar/file.qgs"

        for i in range(10):
            # Push a QGIS project file
            response = self.client.post(
                f"/api/v1/files/{self.project1.id}/{qgis_project_file}/",
                {
                    "file": open(file_path, "rb"),
                },
                format="multipart",
            )
            self.assertTrue(status.is_success(response.status_code))

        jobs = ProcessProjectfileJob.objects.filter(
            project=self.project1,
            status__in=[Job.Status.PENDING, Job.Status.QUEUED, Job.Status.STARTED],
        )

        self.assertEqual(jobs.count(), 1)


# docker-app/qfieldcloud/core/tests/test_queryset.py

import logging

from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core import querysets_utils
from qfieldcloud.core.models import (
    Organization,
    OrganizationMember,
    Project,
    ProjectCollaborator,
    ProjectQueryset,
    Team,
    TeamMember,
    User,
)
from rest_framework.test import APITestCase

from .utils import setup_subscription_plans

logging.disable(logging.CRITICAL)


class QfcTestCase(APITestCase):
    def setUp(self):
        setup_subscription_plans()

        # user1 owns p1 and p2
        # user1 owns o1
        # user1 collaborates on p7
        self.user1 = User.objects.create_user(username="user1", password="abc123")
        self.token1 = AuthToken.objects.get_or_create(user=self.user1)[0]

        # user2 owns p3 and p4
        # user2 admins o1
        self.user2 = User.objects.create_user(username="user2", password="abc123")
        self.token2 = AuthToken.objects.get_or_create(user=self.user2)[0]

        # user2 owns p7 and p8
        # user2 is member of o1
        self.user3 = User.objects.create_user(username="user3", password="abc123")
        self.token3 = AuthToken.objects.get_or_create(user=self.user3)[0]

        # user2 owns no projects
        # user2 is member of o2
        self.user4 = User.objects.create_user(username="user4", password="abc123")
        self.token4 = AuthToken.objects.get_or_create(user=self.user4)[0]

        # organization1 owns p4 and p5
        self.organization1 = Organization.objects.create(
            username="organization1",
            password="abc123",
            user_type=2,
            organization_owner=self.user1,
        )

        # organization2 exists just for confusion
        self.organization2 = Organization.objects.create(
            username="organization2",
            password="abc123",
            user_type=2,
            organization_owner=self.user4,
        )

        self.membership1 = OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user2,
            role=OrganizationMember.Roles.ADMIN,
        )

        self.membership2 = OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user3,
            role=OrganizationMember.Roles.MEMBER,
        )

        self.team1_1 = Team.objects.create(
            username="team1_1",
            password="abc123",
            user_type=User.TYPE_TEAM,
            team_organization=self.organization1,
        )

        self.team2_1 = Team.objects.create(
            username="team2_1",
            password="abc123",
            user_type=User.TYPE_TEAM,
            team_organization=self.organization2,
        )

        self.teammembership1 = TeamMember.objects.create(
            team=self.team1_1,
            member=self.user3,
        )

        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        self.project2 = Project.objects.create(
            name="project2", is_public=True, owner=self.user1
        )

        self.project3 = Project.objects.create(
            name="project3", is_public=False, owner=self.user2
        )

        self.project4 = Project.objects.create(
            name="project4", is_public=True, owner=self.user2
        )

        self.project5 = Project.objects.create(
            name="project5", is_public=False, owner=self.organization1
        )

        self.project6 = Project.objects.create(
            name="project6", is_public=True, owner=self.organization1
        )

        self.project7 = Project.objects.create(
            name="project7", is_public=False, owner=self.user3
        )

        self.project8 = Project.objects.create(
            name="project8", is_public=True, owner=self.user3
        )

        self.project9 = Project.objects.create(
            name="project9", is_public=False, owner=self.organization1
        )

        self.collaborator1 = ProjectCollaborator.objects.create(
            project=self.project7,
            collaborator=self.user1,
            role=ProjectCollaborator.Roles.REPORTER,
        )

        self.collaborator2 = ProjectCollaborator.objects.create(
            project=self.project9,
            collaborator=self.team1_1,
            role=ProjectCollaborator.Roles.EDITOR,
        )

    def test_get_users(self):
        # should get all the available users
        queryset = querysets_utils.get_users("")
        self.assertEqual(len(queryset), 8)
        self.assertTrue(self.user1 in queryset)
        self.assertTrue(self.user2 in queryset)
        self.assertTrue(self.user3 in queryset)
        self.assertTrue(self.organization1.user_ptr in queryset)
        self.assertTrue(self.organization2.user_ptr in queryset)
        self.assertTrue(self.team1_1.user_ptr in queryset)
        self.assertTrue(self.team2_1.user_ptr in queryset)

        # should get all the available users
        queryset = querysets_utils.get_users("user3")
        self.assertEqual(len(queryset), 1)
        self.assertTrue(self.user3 in queryset)

        # should get only the users that are not an organization
        queryset = querysets_utils.get_users("", exclude_organizations=True)
        self.assertEqual(len(queryset), 6)
        self.assertTrue(self.user1 in queryset)
        self.assertTrue(self.user2 in queryset)
        self.assertTrue(self.user3 in queryset)
        self.assertTrue(self.user4 in queryset)
        self.assertTrue(self.team1_1.user_ptr in queryset)
        self.assertTrue(self.team2_1.user_ptr in queryset)

        # should get only the users that are not a team
        queryset = querysets_utils.get_users("", exclude_teams=True)
        self.assertEqual(len(queryset), 6)
        self.assertTrue(self.user1 in queryset)
        self.assertTrue(self.user2 in queryset)
        self.assertTrue(self.user3 in queryset)
        self.assertTrue(self.user4 in queryset)
        self.assertTrue(self.organization1.user_ptr in queryset)
        self.assertTrue(self.organization2.user_ptr in queryset)

        # should get all the users, that are not members or owners of an organization, or teams within the organization
        queryset = querysets_utils.get_users("", organization=self.organization1)
        self.assertEqual(len(queryset), 3)
        self.assertTrue(self.user4 in queryset)
        self.assertTrue(self.organization2.user_ptr in queryset)
        self.assertTrue(self.team1_1.user_ptr in queryset)

        # should get all the users, that are not members or owner of a project
        queryset = querysets_utils.get_users("", project=self.project1)
        self.assertEqual(len(queryset), 5)
        self.assertTrue(self.user2 in queryset)
        self.assertTrue(self.user3 in queryset)
        self.assertTrue(self.user4 in queryset)
        self.assertTrue(self.organization1.user_ptr in queryset)
        self.assertTrue(self.organization2.user_ptr in queryset)

        # should get all the users, that are not members or owner of a project
        queryset = querysets_utils.get_users("", project=self.project5)
        self.assertEqual(len(queryset), 6)
        self.assertTrue(self.user1 in queryset)
        self.assertTrue(self.user2 in queryset)
        self.assertTrue(self.user3 in queryset)
        self.assertTrue(self.user4 in queryset)
        self.assertTrue(self.team1_1.user_ptr in queryset)
        self.assertTrue(self.organization2.user_ptr in queryset)

        # should get all the users, that are not members or owner of a project and are not an organization
        queryset = querysets_utils.get_users(
            "", project=self.project1, exclude_organizations=True
        )
        self.assertEqual(len(queryset), 3)
        self.assertTrue(self.user2 in queryset)
        self.assertTrue(self.user3 in queryset)
        self.assertTrue(self.user4 in queryset)

    def test_projects_roles_and_role_origins(self):
        """
        Checks user_role and user_role_origin are correctly defined
        """

        def p(proj, user):
            return Project.objects.for_user(user).get(pk=proj.pk)

        # fmt: off
        self.assertEqual(p(self.project1, self.user1).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project1, self.user1).user_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project2, self.user1).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project2, self.user1).user_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        with self.assertRaises(Project.DoesNotExist):
            p(self.project3, self.user1)
        self.assertEqual(p(self.project4, self.user1).user_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project4, self.user1).user_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project5, self.user1).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project5, self.user1).user_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONOWNER.value)
        self.assertEqual(p(self.project6, self.user1).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project6, self.user1).user_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONOWNER.value)
        with self.assertRaises(Project.DoesNotExist):
            p(self.project7, self.user1)
        self.assertEqual(p(self.project8, self.user1).user_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project8, self.user1).user_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project9, self.user1).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project9, self.user1).user_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONOWNER.value)

        with self.assertRaises(Project.DoesNotExist):
            p(self.project1, self.user2)
        self.assertEqual(p(self.project2, self.user2).user_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project2, self.user2).user_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project3, self.user2).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project3, self.user2).user_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project4, self.user2).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project4, self.user2).user_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project5, self.user2).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project5, self.user2).user_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONADMIN.value)
        self.assertEqual(p(self.project6, self.user2).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project6, self.user2).user_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONADMIN.value)
        with self.assertRaises(Project.DoesNotExist):
            p(self.project7, self.user2)
        self.assertEqual(p(self.project8, self.user2).user_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project8, self.user2).user_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project9, self.user2).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project9, self.user2).user_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONADMIN.value)

        with self.assertRaises(Project.DoesNotExist):
            p(self.project1, self.user3)
        self.assertEqual(p(self.project2, self.user3).user_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project2, self.user3).user_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        with self.assertRaises(Project.DoesNotExist):
            p(self.project3, self.user3)
        self.assertEqual(p(self.project4, self.user3).user_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project4, self.user3).user_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        with self.assertRaises(Project.DoesNotExist):
            p(self.project5, self.user3)
        self.assertEqual(p(self.project6, self.user3).user_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project6, self.user3).user_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project7, self.user3).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project7, self.user3).user_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project8, self.user3).user_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project8, self.user3).user_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project9, self.user3).user_role, ProjectCollaborator.Roles.EDITOR)
        self.assertEqual(p(self.project9, self.user3).user_role_origin, ProjectQueryset.RoleOrigins.TEAMMEMBER.value)
        # fmt: on

    def test_user_roles_and_role_origins(self):
        """
        Checks project_role and project_role_origin are correctly defined
        """

        def p(proj, user):
            return User.objects.for_project(proj).get(pk=user.pk)

        # fmt: off
        self.assertEqual(p(self.project1, self.user1).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project1, self.user1).project_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project2, self.user1).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project2, self.user1).project_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        with self.assertRaises(User.DoesNotExist):
            p(self.project3, self.user1)
        self.assertEqual(p(self.project4, self.user1).project_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project4, self.user1).project_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project5, self.user1).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project5, self.user1).project_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONOWNER.value)
        self.assertEqual(p(self.project6, self.user1).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project6, self.user1).project_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONOWNER.value)
        with self.assertRaises(User.DoesNotExist):
            p(self.project7, self.user1)
        self.assertEqual(p(self.project8, self.user1).project_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project8, self.user1).project_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project9, self.user1).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project9, self.user1).project_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONOWNER.value)

        with self.assertRaises(User.DoesNotExist):
            p(self.project1, self.user2)
        self.assertEqual(p(self.project2, self.user2).project_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project2, self.user2).project_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project3, self.user2).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project3, self.user2).project_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project4, self.user2).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project4, self.user2).project_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project5, self.user2).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project5, self.user2).project_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONADMIN.value)
        self.assertEqual(p(self.project6, self.user2).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project6, self.user2).project_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONADMIN.value)
        with self.assertRaises(User.DoesNotExist):
            p(self.project7, self.user2)
        self.assertEqual(p(self.project8, self.user2).project_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project8, self.user2).project_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project9, self.user2).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project9, self.user2).project_role_origin, ProjectQueryset.RoleOrigins.ORGANIZATIONADMIN.value)

        with self.assertRaises(User.DoesNotExist):
            p(self.project1, self.user3)
        self.assertEqual(p(self.project2, self.user3).project_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project2, self.user3).project_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        with self.assertRaises(User.DoesNotExist):
            p(self.project3, self.user3)
        self.assertEqual(p(self.project4, self.user3).project_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project4, self.user3).project_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        with self.assertRaises(User.DoesNotExist):
            p(self.project5, self.user3)
        self.assertEqual(p(self.project6, self.user3).project_role, ProjectCollaborator.Roles.READER)
        self.assertEqual(p(self.project6, self.user3).project_role_origin, ProjectQueryset.RoleOrigins.PUBLIC.value)
        self.assertEqual(p(self.project7, self.user3).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project7, self.user3).project_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project8, self.user3).project_role, ProjectCollaborator.Roles.ADMIN)
        self.assertEqual(p(self.project8, self.user3).project_role_origin, ProjectQueryset.RoleOrigins.PROJECTOWNER.value)
        self.assertEqual(p(self.project9, self.user3).project_role, ProjectCollaborator.Roles.EDITOR)
        self.assertEqual(p(self.project9, self.user3).project_role_origin, ProjectQueryset.RoleOrigins.TEAMMEMBER.value)
        # fmt: on

    def test_private_project_memberships(self):
        """Tests for QF-1553 - limit collaboration on private projects"""

        def assert_no_role(p, u):
            """Asserts that user has no role on project"""
            # Test on Users
            with self.assertRaises(User.DoesNotExist):
                User.objects.for_project(p).get(pk=u.pk)

            # Test on Project
            with self.assertRaises(Project.DoesNotExist):
                Project.objects.for_user(u).get(pk=p.pk)

        def assert_role(p, u, expected_role, expected_origin):
            """Asserts that user has give role/origin on project"""
            # Test on Users
            if expected_origin != origins.PUBLIC:
                # The User.objects.for_project queryset is not symetric to Project.objects.for_user
                # because it does not include users that have a role because the project is public.
                user = User.objects.for_project(p).get(pk=u.pk)
                self.assertEqual(user.project_role, expected_role)
                self.assertEqual(user.project_role_origin, expected_origin.value)

            # Test on Project
            project = Project.objects.for_user(u).get(pk=p.pk)
            self.assertEqual(project.user_role, expected_role)
            self.assertEqual(project.user_role_origin, expected_origin.value)

        # Shorthands
        roles = ProjectCollaborator.Roles
        origins = ProjectQueryset.RoleOrigins

        # Initial user setup
        u = User.objects.create(username="u")
        o = Organization.objects.create(username="o", organization_owner=u)
        p = Project.objects.create(name="p", owner=u, is_public=True)

        u1 = User.objects.create(username="u1")

        # A public project is readable
        assert_role(p, u1, roles.READER, origins.PUBLIC)

        # Normal collaboration works on a public project
        ProjectCollaborator.objects.create(
            project=p, collaborator=u1, role=roles.MANAGER
        )
        assert_role(p, u1, roles.MANAGER, origins.COLLABORATOR)

        # If project is made private, the collaboration is invalid
        p.is_public = False
        p.save()
        assert_no_role(p, u1)

        # Making the owner an organisation is not enough as user is not member of that org
        p.owner = o
        p.save()
        assert_no_role(p, u1)

        # As the user must be member of the organisation
        OrganizationMember.objects.create(organization=o, member=u1)
        assert_role(p, u1, roles.MANAGER, origins.COLLABORATOR)


# docker-app/qfieldcloud/core/tests/test_user.py

import logging

from django.utils import timezone
from qfieldcloud.authentication.models import AuthToken
from qfieldcloud.core.models import (
    Organization,
    OrganizationMember,
    Project,
    User,
    UserAccount,
)
from rest_framework import status
from rest_framework.test import APITestCase

from .utils import setup_subscription_plans

logging.disable(logging.CRITICAL)


class QfcTestCase(APITestCase):
    def setUp(self):
        setup_subscription_plans()

        # Create a user
        self.user1 = User.objects.create_user(
            username="user1", password="abc123", email="user1@example.com"
        )
        self.token1 = AuthToken.objects.get_or_create(
            user=self.user1,
            client_type=AuthToken.ClientType.QFIELD,
            user_agent="qfield|dev",
        )[0]

        # Create a second user
        self.user2 = User.objects.create_user(
            username="user2", password="abc123", email="user2@example.com"
        )
        self.token2 = AuthToken.objects.get_or_create(
            user=self.user2,
            client_type=AuthToken.ClientType.QFIELD,
            user_agent="qfield|dev",
        )[0]

        # Create a second user
        self.user3 = User.objects.create_user(
            username="user3", password="abc123", email="user3@example.com"
        )
        self.token3 = AuthToken.objects.get_or_create(
            user=self.user3,
            client_type=AuthToken.ClientType.QFIELD,
            user_agent="qfield|dev",
        )[0]

        # Create an organization
        self.organization1 = Organization.objects.create(
            username="organization1",
            password="abc123",
            user_type=2,
            organization_owner=self.user1,
        )

        self.project1 = Project.objects.create(
            name="project1", is_public=False, owner=self.user1
        )

        # Set user2 as member of organization1
        OrganizationMember.objects.create(
            organization=self.organization1,
            member=self.user2,
            role=OrganizationMember.Roles.MEMBER,
            is_public=True,
        ).save()

    def test_login(self):

        response = self.client.post(
            "/api/v1/auth/login/", {"username": "user1", "password": "abc123"}
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertTrue(isinstance(response.data["token"], str))
        self.assertNotEqual(response.data["token"], self.token1.key)
        self.assertEqual(response.data["username"], "user1")

    def test_login_with_email(self):
        response = self.client.post(
            "/api/v1/auth/login/", {"email": "user1@example.com", "password": "abc123"}
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertTrue(isinstance(response.data["token"], str))
        self.assertNotEqual(response.data["token"], self.token1.key)
        self.assertEqual(response.data["username"], "user1")

    def test_login_qfield_expire_other_token(self):
        self.client.credentials(HTTP_USER_AGENT="qfield|dev")
        response = self.client.post(
            "/api/v1/auth/login/", {"username": "user1", "password": "abc123"}
        )
        self.assertTrue(status.is_success(response.status_code))
        self.assertTrue(isinstance(response.data["token"], str))
        self.assertNotEqual(response.data["token"], self.token1.key)
        self.assertLess(
            AuthToken.objects.get(key=self.token1.key).expires_at, timezone.now()
        )
        self.assertEqual(response.data["username"], "user1")

    def test_login_wrong_password(self):
        response = self.client.post(
            "/api/v1/auth/login/", {"username": "user1", "password": "wrong_password"}
        )
        self.assertTrue(status.is_client_error(response.status_code))
        self.assertFalse("token" in response.data)

    def test_get_user(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get("/api/v1/users/user1/")

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(response.data["username"], "user1")
        self.assertEqual(response.data["user_type"], 1)
        self.assertTrue("email" in response.json())

    def test_get_another_user(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get("/api/v1/users/user2/")

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(response.data["username"], "user2")
        self.assertEqual(response.data["user_type"], 1)
        self.assertFalse("email" in response.json())

    def test_get_organization(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get("/api/v1/users/organization1/")

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(response.data["username"], "organization1")
        self.assertEqual(response.data["user_type"], 2)
        self.assertEqual(response.data["organization_owner"], "user1")
        self.assertEqual(len(response.data["members"]), 1)

    def test_list_users(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get("/api/v1/users/")

        self.assertTrue(status.is_success(response.status_code))

        json = response.json()
        json = sorted(json, key=lambda k: k["username"])

        self.assertEqual(len(json), 4)
        self.assertEqual(json[0]["username"], "organization1")
        self.assertEqual(json[1]["username"], "user1")
        self.assertEqual(json[2]["username"], "user2")
        self.assertEqual(json[3]["username"], "user3")

    def test_list_users_filter_exclude_organization(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get("/api/v1/users/?exclude_organizations=1")

        self.assertTrue(status.is_success(response.status_code))

        json = response.json()
        json = sorted(json, key=lambda k: k["username"])

        self.assertEqual(len(json), 3)
        self.assertEqual(json[0]["username"], "user1")
        self.assertEqual(json[1]["username"], "user2")
        self.assertEqual(json[2]["username"], "user3")

    def test_list_users_filter_for_organization(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get(
            f"/api/v1/users/?organization={self.organization1.username}"
        )

        self.assertTrue(status.is_success(response.status_code))

        json = response.json()
        json = sorted(json, key=lambda k: k["username"])

        self.assertEqual(len(json), 1)
        self.assertEqual(json[0]["username"], "user3")

    def test_list_users_filter_for_project(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get(f"/api/v1/users/?project={self.project1.id}")

        self.assertTrue(status.is_success(response.status_code))

        json = response.json()
        json = sorted(json, key=lambda k: k["username"])

        self.assertEqual(len(json), 3)
        self.assertEqual(json[0]["username"], "organization1")
        self.assertEqual(json[1]["username"], "user2")
        self.assertEqual(json[2]["username"], "user3")

    def test_get_the_authenticated_user(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get("/api/v1/users/user1/")

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(response.data["username"], "user1")
        self.assertEqual(response.data["user_type"], 1)
        self.assertTrue("email" in response.json())

    def test_update_the_authenticated_user(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.patch(
            "/api/v1/users/user1/",
            {
                "first_name": "Charles",
                "last_name": "Darwin",
                "email": "charles@beagle.uk",
            },
        )

        self.assertTrue(status.is_success(response.status_code))
        self.assertEqual(response.data["username"], "user1")
        self.assertEqual(response.data["user_type"], 1)
        self.assertEqual(response.data["first_name"], "Charles")
        self.assertEqual(response.data["last_name"], "Darwin")
        self.assertEqual(response.data["email"], "charles@beagle.uk")

    def test_update_another_user(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.patch(
            "/api/v1/users/user2/",
            {"first_name": "Sasha", "last_name": "Grey", "email": "sasha@grey.org"},
        )

        self.assertEqual(response.status_code, 403)

    def test_logout(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.post("/api/v1/auth/logout/")
        self.assertTrue(status.is_success(response.status_code))

        # The token should not work anymore
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.get("/api/v1/users/user/")
        self.assertFalse(status.is_success(response.status_code))

    def test_api_token_auth(self):
        response = self.client.post(
            "/api/v1/auth/token/", {"username": "user1", "password": "abc123"}
        )

        self.assertTrue(status.is_success(response.status_code))
        self.assertTrue(isinstance(response.data["token"], str))
        self.assertNotEqual(response.data["token"], self.token1.key)
        self.assertEqual(response.data["username"], "user1")

    def test_api_token_auth_after_logout(self):
        response = self.client.post(
            "/api/v1/auth/token/", {"username": "user1", "password": "abc123"}
        )

        self.assertTrue(status.is_success(response.status_code))
        self.assertTrue(isinstance(response.data["token"], str))
        self.assertNotEqual(response.data["token"], self.token1.key)
        self.assertEqual(response.data["username"], "user1")

        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
        response = self.client.post("/api/v1/auth/logout/")
        self.assertTrue(status.is_success(response.status_code))

        # Remove the old token from the headers
        self.client.credentials()

        response = self.client.post(
            "/api/v1/auth/token/", {"username": "user1", "password": "abc123"}
        )

        self.assertTrue(status.is_success(response.status_code))
        # The token should be different from before
        self.assertNotEqual(response.data["token"], self.token1.key)
        self.assertEqual(response.data["username"], "user1")

    def test_user_account_is_created_for_each_user(self):
        self.assertTrue(UserAccount.objects.filter(user=self.user1).exists())
        self.assertTrue(UserAccount.objects.filter(user=self.user2).exists())

    def test_user_organizations(self):
        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)

        response = self.client.get("/api/v1/users/user1/organizations/")

        self.assertTrue(status.is_success(response.status_code))
        payload = response.json()

        self.assertEqual(len(payload), 1)

        organization = payload[0]

        self.assertEquals(organization["username"], self.organization1.username)
        self.assertEquals(organization["user_type"], User.TYPE_ORGANIZATION)
        self.assertEquals(organization["membership_role"], "admin")
        self.assertEquals(organization["membership_role_origin"], "organization_owner")
        self.assertEquals(organization["membership_is_public"], True)


# docker-app/qfieldcloud/core/tests/utils.py

import os

from qfieldcloud.subscription.models import Plan


def testdata_path(path):
    basepath = os.path.dirname(os.path.abspath(__file__))
    return os.path.realpath(os.path.join(basepath, "testdata", path))


def get_filename(response):
    content_disposition = response.headers.get("Content-Disposition")

    if content_disposition:
        parts = content_disposition.split("filename=")

        if len(parts) == 2:
            return parts[1][1:-1]

    return None


def setup_subscription_plans():
    Plan.objects.all().delete()
    Plan.objects.bulk_create(
        [
            Plan(
                code="default_user",
                display_name="default user (autocreated)",
                is_default=True,
                is_public=False,
                user_type=Plan.UserType.USER,
            ),
            Plan(
                code="default_org",
                display_name="default organization (autocreated)",
                is_default=True,
                is_public=False,
                user_type=Plan.UserType.ORGANIZATION,
            ),
        ]
    )


# docker-app/qfieldcloud/core/tests/test_deleteorphanedfiles.py
import io

from django.core.management import call_command
from django.test import TestCase
from qfieldcloud.core.models import Project, User
from qfieldcloud.core.utils import get_project_files_count, get_s3_bucket
from qfieldcloud.core.utils2 import storage


class NormalizeBookTitlesTests(TestCase):
    def setUp(self):
        self.u1 = User.objects.create(username="u1")
        self.projects = []

        get_s3_bucket().objects.filter(Prefix="projects/").delete()

        self.generate_projects(2)

    def generate_projects(self, count: int):
        offset = len(self.projects)
        for i in range(1, count + 1):
            p = Project.objects.create(name=f"p{offset + i}", owner=self.u1)
            self.projects.append(p)
            file = io.BytesIO("Hello world!".encode())
            storage.upload_project_file(p, file, "project.qgs")

    def call_command(self, *args, **kwargs):
        out = io.StringIO()
        call_command(
            "deleteorphanedfiles",
            *args,
            stdout=out,
            stderr=out,
            **kwargs,
        )
        return out.getvalue()

    def test_nothing_to_delete(self):
        self.assertEqual(get_project_files_count(self.projects[0].id), 1)
        self.assertEqual(get_project_files_count(self.projects[1].id), 1)

        out = self.call_command()

        self.assertEqual(
            out.strip(),
            "\n".join(
                [
                    "Checking the last 2 project id(s) from the storage...",
                    "No project files to delete.",
                ]
            ),
        )

        self.assertEqual(get_project_files_count(self.projects[0].id), 1)
        self.assertEqual(get_project_files_count(self.projects[1].id), 1)

    def test_dry_run(self):
        self.assertEqual(get_project_files_count(self.projects[0].id), 1)
        self.assertEqual(get_project_files_count(self.projects[1].id), 1)

        out = self.call_command(dry_run=True)

        self.assertEqual(
            out.strip(),
            "\n".join(
                [
                    "Dry run, no files will be deleted.",
                    "Checking the last 2 project id(s) from the storage...",
                    "No project files to delete.",
                ]
            ),
        )

        self.assertEqual(get_project_files_count(self.projects[0].id), 1)
        self.assertEqual(get_project_files_count(self.projects[1].id), 1)

    def test_delete_files(self):
        project_ids = sorted([str(p.id) for p in self.projects])
        Project.objects.filter(id__in=project_ids).delete()

        self.assertEqual(get_project_files_count(project_ids[0]), 1)
        self.assertEqual(get_project_files_count(project_ids[1]), 1)

        out = self.call_command()

        self.assertEqual(
            out.strip(),
            "\n".join(
                [
                    "Checking the last 2 project id(s) from the storage...",
                    f'Deleting project files for "{project_ids[0]}"...',
                    f'Deleting project files for "{project_ids[1]}"...',
                ]
            ),
        )

        self.assertEqual(get_project_files_count(project_ids[0]), 0)
        self.assertEqual(get_project_files_count(project_ids[1]), 0)

    def test_delete_files_dry_run(self):
        project_ids = sorted([str(p.id) for p in self.projects])
        Project.objects.filter(id__in=project_ids).delete()

        self.assertEqual(get_project_files_count(project_ids[0]), 1)
        self.assertEqual(get_project_files_count(project_ids[1]), 1)

        out = self.call_command(dry_run=True)

        self.assertEqual(
            out.strip(),
            "\n".join(
                [
                    "Dry run, no files will be deleted.",
                    "Checking the last 2 project id(s) from the storage...",
                    f'Deleting project files for "{project_ids[0]}"...',
                    f'Deleting project files for "{project_ids[1]}"...',
                ]
            ),
        )

        self.assertEqual(get_project_files_count(project_ids[0]), 1)
        self.assertEqual(get_project_files_count(project_ids[1]), 1)

    def test_invalid_uuid(self):
        self.assertEqual(get_project_files_count(self.projects[0].id), 1)
        self.assertEqual(get_project_files_count(self.projects[1].id), 1)

        file = io.BytesIO("Hello world!".encode())
        storage.upload_file(file, "projects/strangename/project.qgs")

        out = self.call_command()

        self.assertEqual(
            out.strip(),
            "\n".join(
                [
                    "Invalid uuid: strangename/project.qgs",
                    "Checking the last 2 project id(s) from the storage...",
                    "No project files to delete.",
                ]
            ),
        )

        self.assertEqual(get_project_files_count(self.projects[0].id), 1)
        self.assertEqual(get_project_files_count(self.projects[1].id), 1)

    def test_batches(self):
        self.generate_projects(2)
        project_ids = sorted([str(p.id) for p in self.projects])
        Project.objects.filter(id__in=project_ids[:2]).delete()

        self.assertEqual(get_project_files_count(project_ids[0]), 1)
        self.assertEqual(get_project_files_count(project_ids[1]), 1)
        self.assertEqual(get_project_files_count(project_ids[2]), 1)
        self.assertEqual(get_project_files_count(project_ids[3]), 1)

        out = self.call_command(limit=2)

        self.assertEqual(
            out.strip(),
            "\n".join(
                [
                    "Checking a batch of 2 project ids from the storage...",
                    "Checking a batch of 2 project ids from the storage...",
                    f'Deleting project files for "{project_ids[0]}"...',
                    f'Deleting project files for "{project_ids[1]}"...',
                ]
            ),
        )

        self.assertEqual(get_project_files_count(project_ids[0]), 0)
        self.assertEqual(get_project_files_count(project_ids[1]), 0)
        self.assertEqual(get_project_files_count(project_ids[2]), 1)
        self.assertEqual(get_project_files_count(project_ids[3]), 1)

    def test_deletes_extra_files_on_second_level(self):
        file = io.BytesIO("Hello world!".encode())
        storage.upload_project_file(self.projects[0], file, "inner/path/data.txt")

        project_ids = sorted([str(p.id) for p in self.projects])
        Project.objects.filter(id__in=project_ids).delete()

        self.assertEqual(get_project_files_count(self.projects[0].id), 2)
        self.assertEqual(get_project_files_count(self.projects[1].id), 1)

        out = self.call_command()

        self.assertEqual(
            out.strip(),
            "\n".join(
                [
                    "Checking the last 2 project id(s) from the storage...",
                    f'Deleting project files for "{project_ids[0]}"...',
                    f'Deleting project files for "{project_ids[1]}"...',
                ]
            ),
        )

        self.assertEqual(get_project_files_count(self.projects[0].id), 0)
        self.assertEqual(get_project_files_count(self.projects[1].id), 0)
